From b28cc29961dcabdc89866110c438ce83b9147c2b Mon Sep 17 00:00:00 2001 From: better0fdead Date: Tue, 5 Sep 2023 16:47:03 +0300 Subject: [PATCH] crud: add readview support Added readview support for select and pairs. Closes #343 --- crud.lua | 4 + crud/compare/filters.lua | 89 + crud/compare/plan.lua | 125 ++ crud/readview.lua | 613 +++++++ crud/select/executor.lua | 146 ++ crud/select/merger.lua | 62 + crud/stats/operation.lua | 1 + test/integration/pairs_readview_test.lua | 936 ++++++++++ test/integration/select_readview_test.lua | 2037 +++++++++++++++++++++ 9 files changed, 4013 insertions(+) create mode 100644 crud/readview.lua create mode 100644 test/integration/pairs_readview_test.lua create mode 100644 test/integration/select_readview_test.lua diff --git a/crud.lua b/crud.lua index 7ceac988..3b64745a 100644 --- a/crud.lua +++ b/crud.lua @@ -20,6 +20,7 @@ local borders = require('crud.borders') local sharding_metadata = require('crud.common.sharding.sharding_metadata') local utils = require('crud.common.utils') local stats = require('crud.stats') +local readview = require('crud.readview') local crud = {} @@ -147,6 +148,8 @@ crud.reset_stats = stats.reset -- @function storage_info crud.storage_info = utils.storage_info +crud.readview = readview.call + --- Initializes crud on node -- -- Exports all functions that are used for calls @@ -174,6 +177,7 @@ function crud.init_storage() count.init() borders.init() sharding_metadata.init() + readview.init() _G._crud.storage_info_on_storage = utils.storage_info_on_storage end diff --git a/crud/compare/filters.lua b/crud/compare/filters.lua index 145e0190..046149a6 100644 --- a/crud/compare/filters.lua +++ b/crud/compare/filters.lua @@ -146,6 +146,75 @@ local function parse(space, conditions, opts) return filter_conditions end +local function parse_readview(space_f, conditions, opts) + dev_checks('table','table', '?table', { + scan_condition_num = '?number', + tarantool_iter = 'number', + }) + + conditions = conditions ~= nil and conditions or {} + + local space_format = space_f:format() + local space_indexes = space_f.index + + local fieldnos_by_names = {} + + for i, field_format in ipairs(space_format) do + fieldnos_by_names[field_format.name] = i + end + + local filter_conditions = {} + for i, condition in ipairs(conditions) do + if i ~= opts.scan_condition_num then + -- Index check (including one and multicolumn) + local fields + local fields_types = {} + local values_opts + + local index = space_indexes[condition.operand] + + if index ~= nil then + fields = get_index_fieldnos(index) + fields_types = get_index_fields_types(index) + values_opts = get_values_opts(index) + else + local fieldno = fieldnos_by_names[condition.operand] + + if fieldno ~= nil then + fields = {fieldno} + else + -- We assume this is jsonpath, so it is + -- not in fieldnos_by_name map. + fields = {condition.operand} + end + + local field_format = space_format[fieldno] + local is_nullable + + if field_format ~= nil then + fields_types = {field_format.type} + is_nullable = field_format.is_nullable == true + end + + values_opts = { + {is_nullable = is_nullable, collation = nil}, + } + end + + table.insert(filter_conditions, { + fields = fields, + operator = condition.operator, + values = condition.values, + types = fields_types, + early_exit_is_possible = is_early_exit_possible(index, opts.tarantool_iter, condition), + values_opts = values_opts, + }) + end + end + + return filter_conditions +end + local function format_value(value) if type(value) == 'nil' then return 'nil' @@ -665,6 +734,26 @@ function filters.gen_func(space, conditions, opts) return filter_func end +function filters.gen_func_readview(space, space_format, conditions, opts) + dev_checks('table','table', '?table', { + tarantool_iter = 'number', + scan_condition_num = '?number', + }) + + local filter_conditions, err = parse_readview(space_format, conditions, { + scan_condition_num = opts.scan_condition_num, + tarantool_iter = opts.tarantool_iter, + }) + if err ~= nil then + return nil, GenFiltersError:new("Failed to generate filters for specified conditions: %s", err) + end + + local filter_code = gen_filter_code(filter_conditions) + local filter_func = compile(filter_code) + + return filter_func +end + filters.internal = { parse = parse, gen_filter_code = gen_filter_code, diff --git a/crud/compare/plan.lua b/crud/compare/plan.lua index ffe45596..43e278da 100644 --- a/crud/compare/plan.lua +++ b/crud/compare/plan.lua @@ -286,6 +286,131 @@ function plan.new(space, conditions, opts) return plan end +function plan.new_readview(space, conditions, opts) + dev_checks('table', '?table', { + first = '?number', + after_tuple = '?table|cdata', + field_names = '?table', + force_map_call = '?boolean', + sharding_key_as_index_obj = '?table', + bucket_id = '?number|cdata', + }) + + conditions = conditions ~= nil and conditions or {} + opts = opts or {} + + local space_name = space.name + local space_indexes = space.index + local space_format = space.format + + if space_indexes == nil or next(space_indexes) == nil then + return nil, NoIndexesError:new('Space %q has no indexes, space should have primary index', space_name) + end + + if conditions == nil then -- also cdata + conditions = {} + end + + local scan_index + local scan_iter + local scan_value + local scan_condition_num + + local fieldno_map = utils.get_format_fieldno_map(space_format) + + -- search index to iterate over + for i, condition in ipairs(conditions) do + scan_index = get_index_for_condition(space_indexes, space_format, condition) + + if scan_index ~= nil then + scan_iter = compare_conditions.get_tarantool_iter(condition) + scan_value = condition.values + scan_condition_num = i + break + end + end + + -- default iteration index is primary index + local primary_index = space_indexes[0] + if scan_index == nil then + + if not index_is_allowed(primary_index) then + return nil, IndexTypeError:new('An index that matches specified conditions was not found: ' .. + 'At least one of condition indexes or primary index should be of type TREE') + end + + scan_index = primary_index + scan_iter = box.index.GE -- default iteration is `next greater than previous` + scan_value = {} + end + + local cmp_key_parts = utils.merge_primary_key_parts(scan_index.parts, primary_index.parts) + local field_names = utils.enrich_field_names_with_cmp_key(opts.field_names, cmp_key_parts, space_format) + + -- handle opts.first + local total_tuples_count + local scan_after_tuple, err = construct_after_tuple_by_fields( + fieldno_map, field_names, opts.after_tuple + ) + if err ~= nil then + return nil, err + end + + local is_equal_iter = scan_iter == box.index.EQ or scan_iter == box.index.REQ + if opts.first ~= nil then + total_tuples_count = math.abs(opts.first) + + if opts.first < 0 then + scan_iter = utils.invert_tarantool_iter(scan_iter) + + -- it makes no sence for EQ/REQ + if not is_equal_iter then + -- scan condition becomes border condition + scan_condition_num = nil + + if scan_after_tuple ~= nil then + -- after becomes a new scan value + if has_keydef then + local key_def = keydef_lib.new(scan_index.parts) + scan_value = key_def:extract_key(scan_after_tuple) + else + scan_value = utils.extract_key(scan_after_tuple, scan_index.parts) + end + else + scan_value = nil + end + end + end + end + + local sharding_key = nil + if opts.bucket_id == nil then + local sharding_index = opts.sharding_key_as_index_obj or primary_index + + sharding_key = get_sharding_key_from_scan_value(scan_value, scan_index, scan_iter, sharding_index) + + if sharding_key == nil then + sharding_key = extract_sharding_key_from_conditions(conditions, sharding_index, + space_indexes, fieldno_map) + end + end + + local plan = { + conditions = conditions, + space_name = space_name, + index_id = scan_index.id, + scan_value = scan_value, + after_tuple = scan_after_tuple, + scan_condition_num = scan_condition_num, + tarantool_iter = scan_iter, + total_tuples_count = total_tuples_count, + sharding_key = sharding_key, + field_names = field_names, + } + + return plan +end + plan.internal = { get_sharding_key_from_scan_value = get_sharding_key_from_scan_value, extract_sharding_key_from_conditions = extract_sharding_key_from_conditions diff --git a/crud/readview.lua b/crud/readview.lua new file mode 100644 index 00000000..00896846 --- /dev/null +++ b/crud/readview.lua @@ -0,0 +1,613 @@ +local errors = require('errors') + +local fiber = require('fiber') +local common = require('crud.select.compat.common') + +local Merger = require('crud.select.merger') + +local stash = require('crud.common.stash') +local utils = require('crud.common.utils') +local sharding = require('crud.common.sharding') +local select_executor = require('crud.select.executor') +local select_filters = require('crud.compare.filters') +local dev_checks = require('crud.common.dev_checks') +local schema = require('crud.common.schema') +local const = require('crud.common.const') +local sharding_metadata_module = require('crud.common.sharding.sharding_metadata') +local stats = require('crud.stats') +local checks = require('checks') + +local compare_conditions = require('crud.compare.conditions') +local select_plan = require('crud.compare.plan') + +local ReadviewError = errors.new_class('ReadviewError', {capture_stack = false}) + +local readview = {} + +function checkers.vshard_call_mode(p) + return p == 'write' or p == 'read' +end + +local function readview_open_on_storage(readview_name) + local read_view = box.read_view.open({name = readview_name}) + + if read_view == nil then + return "error creating readview" + end + + local result = {} + result.uuid = box.info().uuid + result.id = read_view.id + + return result +end + +local function readview_close_on_storage(readview_uuid) + dev_checks('table') + + local list = box.read_view.list() + + local readview_id + for _, replica_info in pairs(readview_uuid) do + if replica_info.uuid == box.info().uuid then + readview_id = replica_info.id + end + end + + if readview_id == nil then + return "" + end + + for k,v in pairs(list) do + if v.id == readview_id then + list[k]:close() + return list[k]:info() + end + end + + return "no such readview" +end + +local function select_readview_on_storage(space_name, index_id, conditions, opts, readview_name) + dev_checks('string', 'number', '?table', { + scan_value = 'table', + after_tuple = '?table', + tarantool_iter = 'number', + limit = 'number', + scan_condition_num = '?number', + field_names = '?table', + sharding_key_hash = '?number', + sharding_func_hash = '?number', + skip_sharding_hash_check = '?boolean', + yield_every = '?number', + fetch_latest_metadata = '?boolean', + }, 'number') + + local cursor = {} + if opts.fetch_latest_metadata then + local replica_schema_version + if box.info.schema_version ~= nil then + replica_schema_version = box.info.schema_version + else + replica_schema_version = box.internal.schema_version() + end + cursor.storage_info = { + replica_uuid = box.info().uuid, + replica_schema_version = replica_schema_version, + } + end + + local list = box.read_view.list() + local space + + + for k,v in pairs(list) do + if v.id == readview_name then + space = list[k].space[space_name] + end + end + + if space == nil then + return cursor, ReadviewError:new("Space %q doesn't exist", space_name) + end + space.format = box.space[space_name]:format() + local space_format = box.space[space_name] + if space_format == nil then + return cursor, ReadviewError:new("Space %q doesn't exist", space_name) + end + + + local index = space.index[index_id] + local index_format = space_format.index[index_id] + if index == nil then + return cursor, SelectError:new("Index with ID %s doesn't exist", index_id) + end + + local _, err = sharding.check_sharding_hash(space_name, + opts.sharding_func_hash, + opts.sharding_key_hash, + opts.skip_sharding_hash_check) + + if err ~= nil then + return nil, err + end + + local filter_func, err = select_filters.gen_func(space_format, conditions, { + tarantool_iter = opts.tarantool_iter, + scan_condition_num = opts.scan_condition_num, + }) + if err ~= nil then + return cursor, SelectError:new("Failed to generate tuples filter: %s", err) + end + + -- execute select + local resp, err = select_executor.execute_readview(space, space_format, index, index_format, filter_func, { + scan_value = opts.scan_value, + after_tuple = opts.after_tuple, + tarantool_iter = opts.tarantool_iter, + limit = opts.limit, + yield_every = opts.yield_every, + }) + if err ~= nil then + return cursor, SelectError:new("Failed to execute select: %s", err) + end + + if resp.tuples_fetched < opts.limit or opts.limit == 0 then + cursor.is_end = true + else + local last_tuple = resp.tuples[#resp.tuples] + cursor.after_tuple = last_tuple + end + + cursor.stats = { + tuples_lookup = resp.tuples_lookup, + tuples_fetched = resp.tuples_fetched, + } + + -- getting tuples with user defined fields (if `fields` option is specified) + -- and fields that are needed for comparison on router (primary key + scan key) + local filtered_tuples = schema.filter_tuples_fields(resp.tuples, opts.field_names) + + local result = {cursor, filtered_tuples} + + local select_module_compat_info = stash.get(stash.name.select_module_compat_info) + if not select_module_compat_info.has_merger then + if opts.fetch_latest_metadata then + result[3] = cursor.storage_info.replica_schema_version + end + end + + return unpack(result) +end + +local function build_select_iterator(vshard_router, space_name,readview_uuid, user_conditions, opts) + dev_checks('table', 'string','table', '?table', { + after = '?table|cdata', + first = '?number', + batch_size = '?number', + bucket_id = '?number|cdata', + force_map_call = '?boolean', + field_names = '?table', + yield_every = '?number', + call_opts = 'table', + }) + + opts = opts or {} + + if opts.batch_size ~= nil and opts.batch_size < 1 then + return nil, ReadviewError:new("batch_size should be > 0") + end + + if opts.yield_every ~= nil and opts.yield_every < 1 then + return nil, ReadviewError:new("yield_every should be > 0") + end + + local yield_every = opts.yield_every or const.DEFAULT_YIELD_EVERY + + -- check conditions + local conditions, err = compare_conditions.parse(user_conditions) + if err ~= nil then + return nil, ReadviewError:new("Failed to parse conditions: %s", err) + end + + local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router) + + if err ~= nil then + return nil, ReadviewError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD + end + if space == nil then + return nil, ReadviewError:new("Space %q doesn't exist", space_name), const.NEED_SCHEMA_RELOAD + end + + local sharding_key_data = {} + local sharding_func_hash = nil + local skip_sharding_hash_check = nil + + -- We don't need sharding info if bucket_id specified. + if opts.bucket_id == nil then + sharding_key_data, err = sharding_metadata_module.fetch_sharding_key_on_router(vshard_router, space_name) + if err ~= nil then + return nil, err + end + else + skip_sharding_hash_check = true + end + + -- plan select + local plan, err = select_plan.new(space, conditions, { + first = opts.first, + after_tuple = opts.after, + field_names = opts.field_names, + sharding_key_as_index_obj = sharding_key_data.value, + bucket_id = opts.bucket_id, + }) + + + if err ~= nil then + return nil, ReadviewError:new("Failed to plan select: %s", err), const.NEED_SCHEMA_RELOAD + end + + -- set replicasets to select from + local replicasets_to_select, err = vshard_router:routeall() + if err ~= nil then + return nil, ReadviewError:new("Failed to get router replicasets: %s", err) + end + + -- Whether to call one storage replicaset or perform + -- map-reduce? + -- + -- If map-reduce is requested explicitly, ignore provided + -- bucket_id and fetch data from all storage replicasets. + -- + -- Otherwise: + -- + -- 1. If particular replicaset is pointed by a caller (using + -- the bucket_id option[^1]), crud MUST fetch data only + -- from this storage replicaset: disregarding whether other + -- storages have tuples that fit given condition. + -- + -- 2. If a replicaset may be deduced from conditions + -- (conditions -> sharding key -> bucket id -> replicaset), + -- fetch data only from the replicaset. It does not change + -- the result[^2], but significantly reduces network + -- pressure. + -- + -- 3. Fallback to map-reduce otherwise. + -- + -- [^1]: We can change meaning of this option in a future, + -- see gh-190. But now bucket_id points a storage + -- replicaset, not a virtual bucket.local checks = require('checks') + -- does not make the result less consistent (sounds + -- weird, huh?). + local perform_map_reduce = opts.force_map_call == true or + (opts.bucket_id == nil and plan.sharding_key == nil) + if not perform_map_reduce then + local bucket_id_data, err = sharding.key_get_bucket_id(vshard_router, space_name, + plan.sharding_key, opts.bucket_id) + if err ~= nil then + return nil, err + end + + assert(bucket_id_data.bucket_id ~= nil) + + local err + replicasets_to_select, err = sharding.get_replicasets_by_bucket_id(vshard_router, bucket_id_data.bucket_id) + if err ~= nil then + return nil, err, const.NEED_SCHEMA_RELOAD + end + + sharding_func_hash = bucket_id_data.sharding_func_hash + else + stats.update_map_reduces(space_name) + skip_sharding_hash_check = true + end + + local tuples_limit = opts.first + if tuples_limit ~= nil then + tuples_limit = math.abs(tuples_limit) + end + + -- If opts.batch_size is missed we should specify it to min(tuples_limit, DEFAULT_BATCH_SIZE) + local batch_size + if opts.batch_size == nil then + if tuples_limit ~= nil and tuples_limit < common.DEFAULT_BATCH_SIZE then + batch_size = tuples_limit + else + batch_size = common.DEFAULT_BATCH_SIZE + end + else + batch_size = opts.batch_size + end + + local select_opts = { + scan_value = plan.scan_value, + after_tuple = plan.after_tuple, + tarantool_iter = plan.tarantool_iter, + limit = batch_size, + scan_condition_num = plan.scan_condition_num, + field_names = plan.field_names, + sharding_func_hash = sharding_func_hash, + sharding_key_hash = sharding_key_data.hash, + skip_sharding_hash_check = skip_sharding_hash_check, + yield_every = yield_every, + fetch_latest_metadata = opts.call_opts.fetch_latest_metadata, + } + + + local merger = Merger.new_readview(vshard_router, replicasets_to_select,readview_uuid, space, plan.index_id, + '_crud.select_readview_on_storage', + {space_name, plan.index_id, plan.conditions, select_opts}, + {tarantool_iter = plan.tarantool_iter, field_names = plan.field_names, call_opts = opts.call_opts} + ) + + return { + tuples_limit = tuples_limit, + merger = merger, + plan = plan, + space = space, + netbox_schema_version = netbox_schema_version, + } +end + +local function readview_module_call_xc(vshard_router, space_name,readview_uuid, user_conditions, opts) + checks('table','string','table', '?table', { + after = '?table|cdata', + first = '?number', + batch_size = '?number', + bucket_id = '?number|cdata', + force_map_call = '?boolean', + fields = '?table', + fullscan = '?boolean', + fetch_latest_metadata = '?boolean', + + mode = '?vshard_call_mode', + prefer_replica = '?boolean', + balance = '?boolean', + timeout = '?number', + + vshard_router = '?string|table', + + yield_every = '?number', + }) + + + if opts.first ~= nil and opts.first < 0 then + if opts.after == nil then + return nil, ReadviewError:new("Negative first should be specified only with after option") + end + end + + local iterator_opts = { + after = opts.after, + first = opts.first, + batch_size = opts.batch_size, + bucket_id = opts.bucket_id, + force_map_call = opts.force_map_call, + field_names = opts.fields, + yield_every = opts.yield_every, + call_opts = { + mode = opts.mode, + prefer_replica = opts.prefer_replica, + balance = opts.balance, + timeout = opts.timeout, + fetch_latest_metadata = opts.fetch_latest_metadata, + }, + } + local iter, err = schema.wrap_func_reload( + vshard_router, build_select_iterator, space_name, readview_uuid, user_conditions, iterator_opts + ) + if err ~= nil then + return nil, err + end + common.check_select_safety(space_name, iter.plan, opts) + + local tuples = {} + + local count = 0 + local first = opts.first and math.abs(opts.first) + for _, tuple in iter.merger:pairs() do + if first ~= nil and count >= first then + break + end + + table.insert(tuples, tuple) + count = count + 1 + end + + if opts.first ~= nil and opts.first < 0 then + utils.reverse_inplace(tuples) + end + if opts.fetch_latest_metadata then + -- This option is temporary and is related to [1], [2]. + -- [1] https://github.com/tarantool/crud/issues/236 + -- [2] https://github.com/tarantool/crud/issues/361 + local storages_info = fiber.self().storage.storages_info_on_select + iter = utils.fetch_latest_metadata_for_select(space_name, vshard_router, opts, + storages_info, iter) + end + + -- filter space format by plan.field_names (user defined fields + primary key + scan key) + -- to pass it user as metadata + local filtered_space_format, err = utils.get_fields_format(iter.space:format(), iter.plan.field_names) + if err ~= nil then + return nil, err + end + + + return { + metadata = table.copy(filtered_space_format), + rows = tuples, + } +end + +local Readview_obj = {} +Readview_obj.__index = Readview_obj + +function Readview_obj:select(space_name, user_conditions, opts) + opts = opts or {} + + local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) + if err ~= nil then + return nil, ReadviewError:new(err) + end + + + return ReadviewError:pcall(sharding.wrap_select_method, vshard_router, readview_module_call_xc, + space_name, self.uuid, user_conditions, opts) +end + + +function Readview_obj:pairs(space_name, user_conditions, opts) + checks('table', 'string', '?table', { + after = '?table|cdata', + first = '?number', + batch_size = '?number', + use_tomap = '?boolean', + bucket_id = '?number|cdata', + force_map_call = '?boolean', + fields = '?table', + fetch_latest_metadata = '?boolean', + + mode = '?vshard_call_mode', + prefer_replica = '?boolean', + balance = '?boolean', + timeout = '?number', + + vshard_router = '?string|table', + + yield_every = '?number', + }) + + opts = opts or {} + + if opts.first ~= nil and opts.first < 0 then + error(string.format("Negative first isn't allowed for pairs")) + end + + local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) + if err ~= nil then + error(err) + end + + local iterator_opts = { + after = opts.after, + first = opts.first, + batch_size = opts.batch_size, + bucket_id = opts.bucket_id, + force_map_call = opts.force_map_call, + field_names = opts.fields, + yield_every = opts.yield_every, + call_opts = { + mode = opts.mode, + prefer_replica = opts.prefer_replica, + balance = opts.balance, + timeout = opts.timeout, + fetch_latest_metadata = opts.fetch_latest_metadata, + }, + } + + local iter, err = schema.wrap_func_reload( + vshard_router, build_select_iterator, space_name, self.uuid, user_conditions, iterator_opts + ) + + if err ~= nil then + error(string.format("Failed to generate iterator: %s", err)) + end + + -- filter space format by plan.field_names (user defined fields + primary key + scan key) + -- to pass it user as metadata + local filtered_space_format, err = utils.get_fields_format(iter.space:format(), iter.plan.field_names) + if err ~= nil then + return nil, err + end + + local gen, param, state = iter.merger:pairs() + if opts.use_tomap == true then + gen, param, state = gen:map(function(tuple) + if opts.fetch_latest_metadata then + -- This option is temporary and is related to [1], [2]. + -- [1] https://github.com/tarantool/crud/issues/236 + -- [2] https://github.com/tarantool/crud/issues/361 + local storages_info = fiber.self().storage.storages_info_on_select + iter = utils.fetch_latest_metadata_for_select(space_name, vshard_router, opts, + storages_info, iter) + filtered_space_format, err = utils.get_fields_format(iter.space:format(), iter.plan.field_names) + if err ~= nil then + return nil, err + end + end + local result + result, err = utils.unflatten(tuple, filtered_space_format) + if err ~= nil then + error(string.format("Failed to unflatten next object: %s", err)) + end + return result + end) + end + + if iter.tuples_limit ~= nil then + gen, param, state = gen:take_n(iter.tuples_limit) + end + + return gen, param, state +end + +function readview.init() + _G._crud.readview_open_on_storage = readview_open_on_storage + _G._crud.readview_close_on_storage = readview_close_on_storage + _G._crud.select_readview_on_storage = select_readview_on_storage + end + +function Readview_obj:close() + local opts = {} + local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) + if err ~= nil then + return ReadviewError:new(err) + end + + local results, err = vshard_router:map_callrw('_crud.readview_close_on_storage', {self.uuid}, opts) + + if err ~= nil then + return ReadviewError:new("Failed to call readview_close_on_storage on storage-side: %s", err) + end + + self.info = results + + return "readview was succesfully closed" + +end + +function Readview_obj:create(name) + local readview = {} + setmetatable(readview, Readview_obj) + readview.name = name + return readview + end + +function readview.call(readview_name) + local opts = {} + local vshard_router, err = utils.get_vshard_router_instance(opts.vshard_router) + if err ~= nil then + return nil, ReadviewError:new(err) + end + + local readview_obj = Readview_obj:create(readview_name) + + local results, err = vshard_router:map_callrw('_crud.readview_open_on_storage', {readview_obj.name}, opts) + + if err ~= nil then + return nil, ReadviewError:new("Failed to call readview_open_on_storage on storage-side: %s", err) + end + local uuid = {} + for _, replicaset_results in pairs(results) do + for _, replica_result in pairs(replicaset_results) do + table.insert(uuid, replica_result) + end + end + readview_obj.uuid = uuid + + return readview_obj, err +end + + +return readview \ No newline at end of file diff --git a/crud/select/executor.lua b/crud/select/executor.lua index 7c8c36c6..d205abc1 100644 --- a/crud/select/executor.lua +++ b/crud/select/executor.lua @@ -46,6 +46,35 @@ local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, aft end end +local function scroll_to_after_tuple_readview(gen, space_format, scan_index, tarantool_iter, after_tuple, yield_every) + local primary_index = space_format.index[0] + + local scroll_key_parts = utils.merge_primary_key_parts(scan_index.parts, primary_index.parts) + + local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter) + local scroll_comparator = select_comparators.gen_tuples_comparator(cmp_operator, scroll_key_parts) + + local looked_up_tuples = 0 + while true do + local tuple + gen.state, tuple = gen(gen.param, gen.state) + looked_up_tuples = looked_up_tuples + 1 + + if yield_every ~= nil and looked_up_tuples % yield_every == 0 then + fiber.yield() + end + + if tuple == nil then + return nil + end + + if scroll_comparator(tuple, after_tuple) then + return tuple + end + end +end + + local generate_value if has_keydef then @@ -189,4 +218,121 @@ function executor.execute(space, index, filter_func, opts) return resp end + +function executor.execute_readview(space, space_format, index, index_format, filter_func, opts) + dev_checks('table', 'table', 'table', 'table', 'function', { + scan_value = 'table', + after_tuple = '?table', + tarantool_iter = 'number', + limit = '?number', + yield_every = '?number', + }) + + opts = opts or {} + + local resp = { tuples_fetched = 0, tuples_lookup = 0, tuples = {} } + + if opts.limit == 0 then + return resp + end + + local value = opts.scan_value + if opts.after_tuple ~= nil then + local iter = opts.tarantool_iter + if iter == box.index.EQ or iter == box.index.REQ then + -- we need to make sure that the keys are equal + -- the code is correct even if value is a partial key + local parts = {} + for i, _ in ipairs(value) do + -- the code required for tarantool 1.10.6 at least + table.insert(parts, index_format.parts[i]) + end + + local is_eq = iter == box.index.EQ + local is_after_bigger + if has_keydef then + local key_def = keydef_lib.new(parts) + local cmp = key_def:compare_with_key(opts.after_tuple, value) + is_after_bigger = (is_eq and cmp > 0) or (not is_eq and cmp < 0) + else + local comparator + if is_eq then + comparator = select_comparators.gen_func('<=', parts) + else + comparator = select_comparators.gen_func('>=', parts) + end + local after_key = utils.extract_key(opts.after_tuple, parts) + is_after_bigger = not comparator(after_key, value) + end + if is_after_bigger then + -- it makes no sence to continue + return resp + end + else + local new_value = generate_value(opts.after_tuple, value, index_format.parts, iter) + if new_value ~= nil then + value = new_value + end + end + end + + local tuple + local raw_gen, param, state = index:pairs(value, {iterator = opts.tarantool_iter}) + local gen = fun.wrap(function(param, state) + local next_state, var = raw_gen(param, state) + + if var ~= nil then + resp.tuples_lookup = resp.tuples_lookup + 1 + end + + return next_state, var + end, param, state) + + if opts.after_tuple ~= nil then + local err + tuple, err = scroll_to_after_tuple_readview(gen, + space_format, index_format, opts.tarantool_iter, opts.after_tuple, opts.yield_every) + if err ~= nil then + return nil, ExecuteSelectError:new("Failed to scroll to the after_tuple: %s", err) + end + + if tuple == nil then + return resp + end + end + + if tuple == nil then + gen.state, tuple = gen(gen.param, gen.state) + end + + local looked_up_tuples = 0 + while true do + if tuple == nil then + break + end + + local matched, early_exit = filter_func(tuple) + + if matched then + table.insert(resp.tuples, tuple) + resp.tuples_fetched = resp.tuples_fetched + 1 + + if opts.limit ~= nil and resp.tuples_fetched >= opts.limit then + break + end + elseif early_exit then + break + end + + gen.state, tuple = gen(gen.param, gen.state) + looked_up_tuples = looked_up_tuples + 1 + + if opts.yield_every ~= nil and looked_up_tuples % opts.yield_every == 0 then + fiber.yield() + end + end + + return resp +end + return executor diff --git a/crud/select/merger.lua b/crud/select/merger.lua index 9d53563a..f50956cc 100644 --- a/crud/select/merger.lua +++ b/crud/select/merger.lua @@ -230,6 +230,68 @@ local function new(vshard_router, replicasets, space, index_id, func_name, func_ return merger end + +local function new_readview(vshard_router, replicasets,readview_uuid, space, index_id, func_name, func_args, opts) + opts = opts or {} + local call_opts = opts.call_opts + local mode = call_opts.mode or 'read' + local vshard_call_name = call.get_vshard_call_name(mode, call_opts.prefer_replica, call_opts.balance) + + -- Request a first data chunk and create merger sources. + local merger_sources = {} + for _, replicaset in pairs(replicasets) do + for replica_uuid, replica in pairs(replicaset.replicas) do + for _, value in pairs(readview_uuid) do + if replica_uuid == value.uuid then + -- Perform a request. + local buf = buffer.ibuf() + local net_box_opts = {is_async = true, buffer = buf, skip_header = false} + table.insert(func_args, value.id) + local future = replica.conn:call(func_name, func_args, net_box_opts) + + -- Create a source. + local context = { + net_box_opts = net_box_opts, + buffer = buf, + func_name = func_name, + func_args = func_args, + replicaset = replicaset, + vshard_call_name = vshard_call_name, + timeout = call_opts.timeout, + fetch_latest_metadata = call_opts.fetch_latest_metadata, + space_name = space.name, + vshard_router = vshard_router, + } + + local state = {future = future} + local source = merger_lib.new_buffer_source(fetch_chunk, context, state) + table.insert(merger_sources, source) + end + end + end + end + + -- Trick for performance. + -- + -- No need to create merger, key_def and pass tuples over the + -- merger, when we have only one tuple source. + if #merger_sources == 1 then + return merger_sources[1] + end + + local keydef = Keydef.new(space, opts.field_names, index_id) + -- When built-in merger is used with external keydef, `merger_lib.new(keydef)` + -- fails. It's simply fixed by casting `keydef` to 'struct key_def&'. + keydef = ffi.cast('struct key_def&', keydef) + + local merger = merger_lib.new(keydef, merger_sources, { + reverse = reverse_tarantool_iters[opts.tarantool_iter], + }) + + return merger +end + return { new = new, + new_readview = new_readview, } diff --git a/crud/stats/operation.lua b/crud/stats/operation.lua index 583cf672..8f487454 100644 --- a/crud/stats/operation.lua +++ b/crud/stats/operation.lua @@ -26,4 +26,5 @@ return { COUNT = 'count', -- BORDERS identifies both `min` and `max`. BORDERS = 'borders', + READVIEW = 'readview', } diff --git a/test/integration/pairs_readview_test.lua b/test/integration/pairs_readview_test.lua new file mode 100644 index 00000000..34c3a540 --- /dev/null +++ b/test/integration/pairs_readview_test.lua @@ -0,0 +1,936 @@ +local fio = require('fio') + +local t = require('luatest') + +local crud_utils = require('crud.common.utils') + +local helpers = require('test.helper') +local tarantool = require('tarantool') + +local pgroup = t.group('pairs_readview', { + {engine = 'memtx'}, +}) + +if not crud_utils.tarantool_version_at_least(2, 11, 0) or tarantool.package ~= 'Tarantool Enterprise' then + return +end + +pgroup.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_select'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + env = { + ['ENGINE'] = g.params.engine, + }, + }) + + g.cluster:start() + + g.space_format = g.cluster.servers[2].net_box.space.customers:format() + + g.cluster.main_server.net_box:eval([[ + require('crud').cfg{ stats = true } + ]]) +end) + +pgroup.after_all(function(g) helpers.stop_cluster(g.cluster) end) + +pgroup.before_each(function(g) + helpers.truncate_space_on_cluster(g.cluster, 'customers') +end) + + +pgroup.test_pairs_no_conditions_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local raw_rows = { + {1, 477, 'Elizabeth', 'Jackson', 12, 'New York'}, + {2, 401, 'Mary', 'Brown', 46, 'Los Angeles'}, + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + {4, 1161, 'William', 'White', 81, 'Chicago'}, + } + + -- without conditions and options + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers') do + table.insert(objects, object) + end + foo:close() + return objects + ]]) + t.assert_equals(objects, raw_rows) + + -- with use_tomap=false (the raw tuples returned) + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', nil, {use_tomap = false}) do + table.insert(objects, object) + end + + foo:close() + return objects + ]]) + t.assert_equals(objects, raw_rows) + + -- no after + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', nil, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]]) + + t.assert_equals(objects, customers) + + -- after obj 2 + local after = crud_utils.flatten(customers[2], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local after = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', nil, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {after}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 4})) + + -- after obj 4 (last) + local after = crud_utils.flatten(customers[4], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local after = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', nil, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {after}) + + t.assert_equals(err, nil) + t.assert_equals(#objects, 0) +end + +pgroup.test_ge_condition_with_index_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'>=', 'age', 33}, + } + + -- no after + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 2, 4})) -- in age order + + -- after obj 3 + local after = crud_utils.flatten(customers[3], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, after}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 4})) -- in age order +end + +pgroup.test_le_condition_with_index_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'<=', 'age', 33}, + } + + -- no after + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + foo = crud.readview('foo') + local objects = {} + for _, object in foo:pairs('customers', conditions, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 1})) -- in age order + + -- after obj 3 + local after = crud_utils.flatten(customers[3], g.space_format) + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {after = after, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, after}) + + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1})) -- in age order +end + +pgroup.test_first_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- w/ tomap + local objects, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', nil, {first = 2, use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + return objects + ]]) + t.assert_equals(err, nil) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 2})) + + local tuples, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local tuples = {} + foo = crud.readview('foo') + for _, tuple in foo:pairs('customers', nil, {first = 2}) do + table.insert(tuples, tuple) + end + foo:close() + return tuples + ]]) + t.assert_equals(err, nil) + t.assert_equals(tuples, { + {1, 477, 'Elizabeth', 'Jackson', 12, 'New York'}, + {2, 401, 'Mary', 'Brown', 46, 'Los Angeles'}, + }) +end + +pgroup.test_negative_first_readview = function(g) + local customers = helpers.insert_objects(g, 'customers',{ + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- negative first + t.assert_error_msg_contains("Negative first isn't allowed for pairs", function() + g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + foo:pairs('customers', nil, {first = -10}) + foo:close() + ]]) + end) +end + +pgroup.test_empty_space_readview = function(g) + local count = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local count = 0 + foo = crud.readview('foo') + for _, object in foo:pairs('customers') do + count = count + 1 + end + foo:close() + return count + ]]) + t.assert_equals(count, 0) +end + +pgroup.test_luafun_compatibility_readview = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + local count = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + local count = foo:pairs('customers'):map(function() return 1 end):sum() + foo:close() + return count + ]]) + t.assert_equals(count, 3) + + count = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + local count = foo:pairs('customers', + {use_tomap = true}):map(function() return 1 end):sum() + foo:close() + return count + ]]) + t.assert_equals(count, 3) +end + +pgroup.test_pairs_partial_result_readview = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + -- condition by indexed non-unique non-primary field (age): + local conditions = {{'>=', 'age', 33}} + + -- condition field is not in opts.fields + local fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + local expected_customers = { + {id = 3, age = 33, name = "David", city = "Los Angeles"}, + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + foo = crud.readview('foo') + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + expected_customers = { + {id = 3, age = 33, name = "David"}, + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + foo = crud.readview('foo') + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- condition by non-indexed non-unique non-primary field (city): + conditions = {{'>=', 'city', 'Lo'}} + + -- condition field is not in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", age = 12}, + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + foo = crud.readview('foo') + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", city = "Los Angeles"}, + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + foo = crud.readview('foo') + + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + foo = crud.readview('foo') + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + + local objects = {} + for _, object in foo:pairs('customers', conditions, {after = tuples[1], use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) +end + +pgroup.test_pairs_cut_result_readview = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + -- condition by indexed non-unique non-primary field (age): + local conditions = {{'>=', 'age', 33}} + + -- condition field is not in opts.fields + local fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + local expected_customers = { + {name = "David", city = "Los Angeles"}, + {name = "Mary", city = "London"}, + {name = "William", city = "Chicago"}, + } + + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local objects = {} + foo = crud.readview('foo') + + for _, object in foo:pairs('customers', conditions, {use_tomap = true, fields = fields}) do + table.insert(objects, object) + end + foo:close() + + return crud.cut_objects(objects, fields) + ]], {conditions, fields}) + t.assert_equals(objects, expected_customers) + + -- without use_tomap + expected_customers = { + {"David", "Los Angeles"}, + {"Mary", "London"}, + {"William", "Chicago"}, + } + + local tuples = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields = ... + + local tuples = {} + foo = crud.readview('foo') + for _, tuple in foo:pairs('customers', conditions, {fields = fields}) do + table.insert(tuples, tuple) + end + foo:close() + + return crud.cut_rows(tuples, nil, fields) + ]], {conditions, fields}) + t.assert_equals(tuples.metadata, nil) + t.assert_equals(tuples.rows, expected_customers) +end + +pgroup.test_pairs_force_map_call_readview = function(g) + local key = 1 + + local first_bucket_id = g.cluster.main_server.net_box:eval([[ + local vshard = require('vshard') + + local key = ... + return vshard.router.bucket_id_strcrc32(key) + ]], {key}) + + local second_bucket_id, err = helpers.get_other_storage_bucket_id(g.cluster, first_bucket_id) + + t.assert_equals(err, nil) + + local customers = helpers.insert_objects(g, 'customers', { + { + id = key, bucket_id = first_bucket_id, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = key, bucket_id = second_bucket_id, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + + local conditions = {{'==', 'id', key}} + + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {use_tomap = true}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions}) + + t.assert_equals(err, nil) + t.assert_equals(#objects, 1) + + objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', conditions, {use_tomap = true, force_map_call = true}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]], {conditions}) + table.sort(objects, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + t.assert_equals(objects, customers) +end + +pgroup.test_pairs_timeout_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local raw_rows = { + {1, 477, 'Elizabeth', 'Jackson', 12, 'New York'}, + {2, 401, 'Mary', 'Brown', 46, 'Los Angeles'}, + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + {4, 1161, 'William', 'White', 81, 'Chicago'}, + } + + local objects = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', nil, {timeout = 1}) do + table.insert(objects, object) + end + foo:close() + + return objects + ]]) + t.assert_equals(objects, raw_rows) +end + +pgroup.test_opts_not_damaged = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + -- bucket_id is 477, storage is s-2 + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + -- bucket_id is 401, storage is s-2 + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + -- bucket_id is 2804, storage is s-1 + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + -- bucket_id is 1161, storage is s-2 + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local expected_customers = { + {id = 4, name = "William", age = 46}, + } + + -- after tuple should be in `fields` format + primary key + local fields = {'name', 'age'} + local after = {"Mary", 46, 2} + + local pairs_opts = { + timeout = 1, bucket_id = 1161, + batch_size = 105, first = 2, after = after, + fields = fields, mode = 'read', prefer_replica = false, + balance = false, force_map_call = false, use_tomap = true, + } + local new_pairs_opts, objects = g.cluster.main_server:eval([[ + local crud = require('crud') + + local pairs_opts = ... + + local objects = {} + foo = crud.readview('foo') + for _, object in foo:pairs('customers', nil, pairs_opts) do + table.insert(objects, object) + end + foo:close() + + return pairs_opts, objects + ]], {pairs_opts}) + + t.assert_equals(objects, expected_customers) + t.assert_equals(new_pairs_opts, pairs_opts) +end + +-- gh-220: bucket_id argument is ignored when it cannot be deduced +-- from provided select/pairs conditions. +pgroup.test_pairs_no_map_reduce_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + -- bucket_id is 477, storage is s-2 + id = 1, name = 'Elizabeth', last_name = 'Jackson', + age = 12, city = 'New York', + }, { + -- bucket_id is 401, storage is s-2 + id = 2, name = 'Mary', last_name = 'Brown', + age = 46, city = 'Los Angeles', + }, { + -- bucket_id is 2804, storage is s-1 + id = 3, name = 'David', last_name = 'Smith', + age = 33, city = 'Los Angeles', + }, { + -- bucket_id is 1161, storage is s-2 + id = 4, name = 'William', last_name = 'White', + age = 81, city = 'Chicago', + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local router = g.cluster:server('router').net_box + local map_reduces_before = helpers.get_map_reduces_stat(router, 'customers') + + -- Case: no conditions, just bucket id. + local rows = router:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + + local rows = foo:pairs(...):totable() + foo:close() + return rows + ]], { + 'customers', + nil, + {bucket_id = 2804, timeout = 1}, + }) + t.assert_equals(rows, { + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + }) + + local map_reduces_after_1 = helpers.get_map_reduces_stat(router, 'customers') + local diff_1 = map_reduces_after_1 - map_reduces_before + t.assert_equals(diff_1, 0, 'Select request was not a map reduce') + + -- Case: EQ on secondary index, which is not in the sharding + -- index (primary index in the case). + local rows = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + local rows = foo:pairs(...):totable() + foo:close() + return rows + ]], { + 'customers', + {{'==', 'age', 81}}, + {bucket_id = 1161, timeout = 1}, + }) + t.assert_equals(rows, { + {4, 1161, 'William', 'White', 81, 'Chicago'}, + }) + + local map_reduces_after_2 = helpers.get_map_reduces_stat(router, 'customers') + local diff_2 = map_reduces_after_2 - map_reduces_after_1 + t.assert_equals(diff_2, 0, 'Select request was not a map reduce') +end diff --git a/test/integration/select_readview_test.lua b/test/integration/select_readview_test.lua new file mode 100644 index 00000000..0e402720 --- /dev/null +++ b/test/integration/select_readview_test.lua @@ -0,0 +1,2037 @@ +local fio = require('fio') + +local t = require('luatest') + +local crud = require('crud') +local crud_utils = require('crud.common.utils') + + +local helpers = require('test.helper') +local tarantool = require('tarantool') + +local pgroup = t.group('select_readview', { + {engine = 'memtx'}, +}) + +if not crud_utils.tarantool_version_at_least(2, 11, 0) or tarantool.package ~= 'Tarantool Enterprise' then + return +end + + +pgroup.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_select'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + env = { + ['ENGINE'] = g.params.engine, + }, + }) + + g.cluster:start() + + g.space_format = g.cluster.servers[2].net_box.space.customers:format() + + g.cluster:server('router').net_box:eval([[ + require('crud').cfg{ stats = true } + ]]) + g.cluster:server('router').net_box:eval([[ + require('crud.ratelimit').disable() + ]]) +end) + +pgroup.after_all(function(g) helpers.stop_cluster(g.cluster) end) + +pgroup.before_each(function(g) + helpers.truncate_space_on_cluster(g.cluster, 'customers') + helpers.truncate_space_on_cluster(g.cluster, 'developers') + helpers.truncate_space_on_cluster(g.cluster, 'cars') +end) + + + +pgroup.test_non_existent_space_readview = function(g) + -- insert + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + + local result, err = foo:select('non_existent_space', nil, {fullscan=true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, "Space \"non_existent_space\" doesn't exist") +end + +pgroup.test_select_no_index_readview = function(g) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + + local result, err = foo:select('no_index_space', nil, {fullscan=true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, "Space \"no_index_space\" has no indexes, space should have primary index") +end + +pgroup.test_not_valid_value_type_readview = function(g) + local conditions = { + {'=', 'id', 'not_number'} + } + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local conditions = ... + + foo = crud.readview('foo') + + local result, err = foo:select('customers', conditions) + + foo:close() + + return result, err + ]], {conditions}) + + t.assert_equals(obj, nil) + t.assert_str_contains(err.err, "Supplied key type of part 0 does not match index part type: expected unsigned") +end + +pgroup.test_select_readview_all = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + + local result, err = foo:select('customers', nil, {fullscan = true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_readview_with_same_name = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo = crud.readview('foo') + boo = crud.readview('foo') + foo:close() + + local result, err = boo:select('customers', nil, {fullscan = true}) + + boo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_select_readview_without_name = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + boo = crud.readview('') + + local result, err = boo:select('customers', nil, {fullscan = true}) + + boo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) + +end + +pgroup.test_pairs_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local obj, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + local tuples = {} + boo = crud.readview('') + for _, tuple in boo:pairs('customers', {{'<=', 'age', 35}},{use_tomap=false}) do + table.insert(tuples, tuple) + end + + boo:close() + return tuples + ]]) + + t.assert_equals(err, nil) + t.assert_equals(obj, { + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + }) + +end + +pgroup.test_select_readview_all_with_batch_size = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, { + id = 5, name = "Jack", last_name = "Sparrow", + age = 35, city = "London", + }, { + id = 6, name = "William", last_name = "Terner", + age = 25, city = "Oxford", + }, { + id = 7, name = "Elizabeth", last_name = "Swan", + age = 18, city = "Cambridge", + }, { + id = 8, name = "Hector", last_name = "Barbossa", + age = 45, city = "London", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- batch size 1 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', nil, {batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, customers) + + -- batch size 3 + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + bar, err = crud.readview('bar') + + local result, err = bar:select('customers', nil, {batch_size=3, fullscan = true}) + + bar:close() + return result, err + ]]) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, customers) +end + +pgroup.test_eq_condition_with_index_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 33, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "Smith", + age = 81, city = "Chicago", + },{ + id = 5, name = "Hector", last_name = "Barbossa", + age = 33, city = "Chicago", + },{ + id = 6, name = "William", last_name = "White", + age = 81, city = "Chicago", + },{ + id = 7, name = "Jack", last_name = "Sparrow", + age = 33, city = "Chicago", + }, { + id = 8, name = "Nick", last_name = "Smith", + age = 20, city = "London", + } + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'==', 'age_index', 33}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3, 5, 7})) -- in id order + + -- after obj 3 + local after = crud_utils.flatten(customers[3], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 7})) -- in id order + + -- after obj 5 with negative first + local after = crud_utils.flatten(customers[5], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, first = -10}) + + foo:close() + return result, err + ]], {conditions, after}) + + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3})) -- in id order + + -- after obj 8 + local after = crud_utils.flatten(customers[8], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, first = 10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3, 5, 7})) -- in id order + + -- after obj 8 with negative first + local after = crud_utils.flatten(customers[8], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, first = -10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {})) + + -- after obj 2 + local after = crud_utils.flatten(customers[2], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, first = 10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {})) + + -- after obj 2 with negative first + local after = crud_utils.flatten(customers[2], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, first = -10}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 3, 5, 7})) -- in id order +end + +pgroup.test_lt_condition_with_index_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'<', 'age_index', 33}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1})) -- in age order + + -- after obj 1 + local after = crud_utils.flatten(customers[1], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, fullscan=true}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {})) -- in age order +end + +pgroup.test_multiple_conditions_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Rodriguez", + age = 20, city = "Los Angeles", + }, { + id = 2, name = "Elizabeth", last_name = "Rodriguez", + age = 44, city = "Chicago", + }, { + id = 3, name = "Elizabeth", last_name = "Rodriguez", + age = 22, city = "New York", + }, { + id = 4, name = "David", last_name = "Brown", + age = 23, city = "Los Angeles", + }, { + id = 5, name = "Elizabeth", last_name = "Rodriguez", + age = 39, city = "Chicago", + } + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'>', 'age', 20}, + {'==', 'name', 'Elizabeth'}, + {'==', 'city', 'Chicago'}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 2})) -- in age order + + -- after obj 5 + local after = crud_utils.flatten(customers[5], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, fullscan=true}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2})) -- in age order +end + +pgroup.test_composite_index_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Rodriguez", + age = 20, city = "Los Angeles", + }, { + id = 2, name = "Elizabeth", last_name = "Johnson", + age = 44, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Brown", + age = 23, city = "Chicago", + }, { + id = 4, name = "Jessica", last_name = "Jones", + age = 22, city = "New York", + } + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = { + {'>=', 'full_name', {"Elizabeth", "Jo"}}, + } + + -- no after + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 1, 4})) -- in full_name order + + -- after obj 2 + local after = crud_utils.flatten(customers[2], g.space_format) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{after=after, fullscan=true}) + + foo:close() + return result, err + ]], {conditions, after}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {1, 4})) -- in full_name order + + -- partial value in conditions + local conditions = { + {'==', 'full_name', "Elizabeth"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 1})) -- in full_name order + + -- first 1 + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{first=1}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2})) -- in full_name order + + -- first 1 with full specified key + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', {{'==', 'full_name', {'Elizabeth', 'Johnson'}}}, {first = 1}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2})) -- in full_name order +end + +pgroup.test_composite_primary_index_readview = function(g) + local book_translation = helpers.insert_objects(g, 'book_translation', { + { + id = 5, + language = 'Ukrainian', + edition = 55, + translator = 'Mitro Dmitrienko', + comments = 'Translation 55', + } + }) + t.assert_equals(#book_translation, 1) + + local conditions = {{'=', 'id', {5, 'Ukrainian', 55}}} + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('book_translation', conditions) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('book_translation', conditions, {first = 2}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('book_translation', conditions, {first = 1}) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after = ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('book_translation', conditions, {first = 1, after = after}) + + foo:close() + return result, err + ]], {conditions, result.rows[1]}) + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 0) +end + +pgroup.test_select_with_batch_size_1_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, { + id = 5, name = "Jack", last_name = "Sparrow", + age = 35, city = "London", + }, { + id = 6, name = "William", last_name = "Terner", + age = 25, city = "Oxford", + }, { + id = 7, name = "Elizabeth", last_name = "Swan", + age = 18, city = "Cambridge", + }, { + id = 8, name = "Hector", last_name = "Barbossa", + age = 45, city = "London", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- LE + local conditions = {{'<=', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 3, 6, 7, 1})) + + -- LT + local conditions = {{'<', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 6, 7, 1})) + + -- GE + local conditions = {{'>=', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {5, 8, 2, 4})) + + -- GT + local conditions = {{'>', 'age', 35}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{batch_size=1, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {8, 2, 4})) +end + +pgroup.test_select_by_full_sharding_key_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local conditions = {{'==', 'id', 3}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3})) +end + +pgroup.test_select_with_collations_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Oxford", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "oxford", + }, { + id = 3, name = "elizabeth", last_name = "brown", + age = 46, city = "Oxford", + }, { + id = 4, name = "Jack", last_name = "Sparrow", + age = 35, city = "oxford", + }, { + id = 5, name = "William", last_name = "Terner", + age = 25, city = "Oxford", + }, { + id = 6, name = "elizabeth", last_name = "Brown", + age = 33, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- full name index - unicode ci collation (case-insensitive) + local conditions = {{'==', 'name', "Elizabeth"}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {3, 6, 1})) + + -- city - no collation (case-sensitive) + local conditions = {{'==', 'city', "oxford"}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(customers, {2, 4})) +end + +pgroup.test_multipart_primary_index_readview = function(g) + local coords = helpers.insert_objects(g, 'coord', { + { x = 0, y = 0 }, -- 1 + { x = 0, y = 1 }, -- 2 + { x = 0, y = 2 }, -- 3 + { x = 1, y = 3 }, -- 4 + { x = 1, y = 4 }, -- 5 + }) + + local conditions = {{'=', 'primary', 0}} + local result_0, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions) + + foo:close() + return result, err + ]], {conditions}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result_0.rows, result_0.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2, 3})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions,{after = after}) + + foo:close() + return result, err + ]], {conditions, result_0.rows[1]}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {2, 3})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions,{after = after, first = -2}) + + foo:close() + return result, err + ]], {conditions, result_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2})) + + local new_conditions = {{'=', 'y', 1}, {'=', 'primary', 0}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions,{after = after, first = -2}) + + foo:close() + return result, err + ]], {new_conditions, result_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {2})) + + local conditions = {{'=', 'primary', {0, 2}}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions) + + foo:close() + return result, err + ]], {conditions, result_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {3})) + + local conditions_ge = {{'>=', 'primary', 0}} + local result_ge_0, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions,{fullscan=true}) + + foo:close() + return result, err + ]], {conditions_ge}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result_ge_0.rows, result_ge_0.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2, 3, 4, 5})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions,{after = after,fullscan = true}) + + foo:close() + return result, err + ]], {conditions_ge, result_ge_0.rows[1]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {2, 3, 4, 5})) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('coord', conditions,{after = after,first = -3}) + + foo:close() + return result, err + ]], {conditions_ge, result_ge_0.rows[3]}) + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(coords, {1, 2})) +end + +pgroup.test_select_partial_result_bad_input_readview = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + local conditions = {{'>=', 'age', 33}} + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = {'id', 'mame'}, fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(result, nil) + t.assert_str_contains(err.err, 'Space format doesn\'t contain field named "mame"') +end + +pgroup.test_select_partial_result_readview = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + -- condition by indexed non-unique non-primary field (age): + local conditions = {{'>=', 'age', 33}} + + -- condition field is not in opts.fields + local fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + local expected_customers = { + {id = 3, age = 33, name = "David", city = "Los Angeles"}, + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary", city = "London"}, + {id = 4, age = 46, name = "William", city = "Chicago"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by field+primary + -- in age + id order + expected_customers = { + {id = 3, age = 33, name = "David"}, + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, age = 46, name = "Mary"}, + {id = 4, age = 46, name = "William"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- condition by non-indexed non-unique non-primary field (city): + conditions = {{'>=', 'city', 'Lo'}} + + -- condition field is not in opts.fields + fields = {'name', 'age'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", age = 12}, + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", age = 46}, + {id = 3, name = "David", age = 33}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- condition field is in opts.fields + fields = {'name', 'city'} + + -- result doesn't contain primary key, result tuples are sorted by primary + -- in id order + expected_customers = { + {id = 1, name = "Elizabeth", city = "Los Angeles"}, + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- same case with after option + expected_customers = { + {id = 2, name = "Mary", city = "London"}, + {id = 3, name = "David", city = "Los Angeles"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields, after= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, after = after, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields, result.rows[1]}) + + t.assert_equals(err, nil) + objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) +end + +pgroup.test_cut_selected_rows_readview = function(g) + helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + -- condition by indexed non-unique non-primary field (age): + local conditions = {{'>=', 'age', 33}} + + -- condition field is not in opts.fields + local fields = {'name', 'city'} + + local expected_customers = { + {name = "David", city = "Los Angeles"}, + {name = "Mary", city = "London"}, + {name = "William", city = "Chicago"}, + } + + -- with fields option + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions, fields= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fields = fields, fullscan = true}) + + foo:close() + return result, err + ]], {conditions, fields}) + + t.assert_equals(err, nil) + + result, err = g.cluster.main_server.net_box:call('crud.cut_rows', {result.rows, result.metadata, fields}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) + + -- without fields option + + -- fields should be in metadata order if we want to work with cut_rows + fields = {'id', 'bucket_id', 'name'} + + expected_customers = { + {bucket_id = 2804, id = 3, name = "David"}, + {bucket_id = 401, id = 2, name = "Mary"}, + {bucket_id = 1161, id = 4, name = "William"}, + } + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + local conditions= ... + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', conditions,{fullscan = true}) + + foo:close() + return result, err + ]], {conditions}) + + t.assert_equals(err, nil) + + result, err = g.cluster.main_server.net_box:call('crud.cut_rows', {result.rows, result.metadata, fields}) + + t.assert_equals(err, nil) + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, expected_customers) +end + +pgroup.test_select_force_map_call_readview = function(g) + local key = 1 + + local first_bucket_id = g.cluster.main_server.net_box:eval([[ + local vshard = require('vshard') + + local key = ... + return vshard.router.bucket_id_strcrc32(key) + ]], {key}) + + local second_bucket_id, err = helpers.get_other_storage_bucket_id(g.cluster, first_bucket_id) + + t.assert_equals(err, nil) + + local customers = helpers.insert_objects(g, 'customers', { + { + id = key, bucket_id = first_bucket_id, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = key, bucket_id = second_bucket_id, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', {{'==', 'id', 1}}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(#result.rows, 1) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', {{'==', 'id', 1}}, {force_map_call = true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + table.sort(objects, function(obj1, obj2) return obj1.bucket_id < obj2.bucket_id end) + t.assert_equals(objects, customers) +end + +pgroup.test_jsonpath_readview = function(g) + helpers.insert_objects(g, 'developers', { + { + id = 1, name = "Alexey", last_name = "Smith", + age = 20, additional = { a = { b = 140 } }, + }, { + id = 2, name = "Sergey", last_name = "Choppa", + age = 21, additional = { a = { b = 120 } }, + }, { + id = 3, name = "Mikhail", last_name = "Crossman", + age = 42, additional = {}, + }, { + id = 4, name = "Pavel", last_name = "White", + age = 51, additional = { a = { b = 50 } }, + }, { + id = 5, name = "Tatyana", last_name = "May", + age = 17, additional = { a = 55 }, + }, + }) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('developers', {{'>=', '[5]', 40}}, + {fields = {'name', 'last_name'}, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + {id = 3, name = "Mikhail", last_name = "Crossman"}, + {id = 4, name = "Pavel", last_name = "White"}, + } + t.assert_equals(objects, expected_objects) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('developers', {{'<', '["age"]', 21}}, + {fields = {'name', 'last_name'}, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + {id = 1, name = "Alexey", last_name = "Smith"}, + {id = 5, name = "Tatyana", last_name = "May"}, + } + t.assert_equals(objects, expected_objects) +end + +pgroup.test_jsonpath_index_field_readview = function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 2, + manufacturer = 'VAG', + data = {car = { model = 'BMW', color = 'Black' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 5, + manufacturer = 'FIAT', + data = {car = { model = 'Cadillac', color = 'White' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 17, + manufacturer = 'Ford', + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'Mercedes', color = 'Yellow' }}, + }, + }) + + -- PK jsonpath index + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('cars', {{'<=', 'id_ind', 3}, {'<=', 'age', 5}}, + {fields = {'id', 'age'}, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 2}}, + age = 5, + }, + { + id = {car_id = {signed = 1}}, + age = 2, + }} + + t.assert_equals(objects, expected_objects) + + -- Secondary jsonpath index (partial) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('cars', {{'==', 'data_index', 'Yellow'}}, {fields = {'id', 'age'}}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 3}}, + age = 17, + data = {car = { model = 'BMW', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) + + -- Secondary jsonpath index (full) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('cars', {{'==', 'data_index', {'Yellow', 'Mercedes'}}}, {fields = {'id', 'age'}}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + local expected_objects = { + { + id = {car_id = {signed = 4}}, + age = 3, + data = {car = { model = 'Mercedes', color = 'Yellow' }} + }} + + t.assert_equals(objects, expected_objects) +end + +pgroup.test_jsonpath_index_field_pagination_readview = function(g) + t.skip_if( + not crud_utils.tarantool_supports_jsonpath_indexes(), + "Jsonpath indexes supported since 2.6.3/2.7.2/2.8.1" + ) + + local cars = helpers.insert_objects(g, 'cars', { + { + id = {car_id = {signed = 1}}, + age = 5, + manufacturer = 'VAG', + data = {car = { model = 'A', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 2}}, + age = 17, + manufacturer = 'FIAT', + data = {car = { model = 'B', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 3}}, + age = 5, + manufacturer = 'Ford', + data = {car = { model = 'C', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 4}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'D', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 5}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'E', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 6}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'F', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 7}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'G', color = 'Yellow' }}, + }, + { + id = {car_id = {signed = 8}}, + age = 3, + manufacturer = 'General Motors', + data = {car = { model = 'H', color = 'Yellow' }}, + }, + }) + + + -- Pagination (primary index) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('cars', nil, {first = 2}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + local after = ... + + local result, err = foo:select('cars', nil, {first = 2, after = after}) + + foo:close() + return result, err + ]], {result.rows[2]}) + + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {3, 4})) + + -- Reverse pagination (primary index) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + local after = ... + + local result, err = foo:select('cars', nil, {first = -2, after = after}) + + foo:close() + return result, err + ]], {result.rows[1]}) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) + + -- Pagination (secondary index - 1 field) + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('cars', {{'==', 'data_index', 'Yellow'}}, {first = 2}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + + local objects = crud.unflatten_rows(result.rows, result.metadata) + t.assert_equals(objects, helpers.get_objects_by_idxs(cars, {1, 2})) +end + +pgroup.test_select_timeout_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "New York", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "Los Angeles", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 81, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', nil, {timeout = 1, fullscan = true}) + + foo:close() + return result, err + ]]) + + t.assert_equals(err, nil) + t.assert_equals(result.rows, { + {1, 477, "Elizabeth", "Jackson", 12, "New York"}, + {2, 401, "Mary", "Brown", 46, "Los Angeles"}, + {3, 2804, "David", "Smith", 33, "Los Angeles"}, + {4, 1161, "William", "White", 81, "Chicago"}, + }) +end + +pgroup.test_opts_not_damaged_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + id = 1, name = "Elizabeth", last_name = "Jackson", + age = 12, city = "Los Angeles", + }, { + id = 2, name = "Mary", last_name = "Brown", + age = 46, city = "London", + }, { + id = 3, name = "David", last_name = "Smith", + age = 33, city = "Los Angeles", + }, { + id = 4, name = "William", last_name = "White", + age = 46, city = "Chicago", + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + -- after tuple should be in `fields` format + primary key + local fields = {'name', 'age'} + local after = {"Mary", 46, 2} + + local select_opts = { + timeout = 1, bucket_id = 1161, + batch_size = 105, first = 2, after = after, + fields = fields, mode = 'read', prefer_replica = false, + balance = false, force_map_call = false, + } + local new_select_opts, err = g.cluster.main_server:eval([[ + local crud = require('crud') + + local select_opts = ... + + foo, err = crud.readview('foo') + + local _, err = foo:select('customers', nil, select_opts) + + foo:close() + + return select_opts, err + ]], {select_opts}) + + t.assert_equals(err, nil) + t.assert_equals(new_select_opts, select_opts) +end + +-- gh-220: bucket_id argument is ignored when it cannot be deduced +-- from provided select/pairs conditions. +pgroup.test_select_no_map_reduce_readview = function(g) + local customers = helpers.insert_objects(g, 'customers', { + { + -- bucket_id is 477, storage is s-2 + id = 1, name = 'Elizabeth', last_name = 'Jackson', + age = 12, city = 'New York', + }, { + -- bucket_id is 401, storage is s-2 + id = 2, name = 'Mary', last_name = 'Brown', + age = 46, city = 'Los Angeles', + }, { + -- bucket_id is 2804, storage is s-1 + id = 3, name = 'David', last_name = 'Smith', + age = 33, city = 'Los Angeles', + }, { + -- bucket_id is 1161, storage is s-2 + id = 4, name = 'William', last_name = 'White', + age = 81, city = 'Chicago', + }, + }) + + table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end) + + local router = g.cluster:server('router').net_box + local map_reduces_before = helpers.get_map_reduces_stat(router, 'customers') + + -- Case: no conditions, just bucket id. + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', nil, {bucket_id = 2804, timeout = 1, fullscan = true}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + t.assert_equals(result.rows, { + {3, 2804, 'David', 'Smith', 33, 'Los Angeles'}, + }) + + local map_reduces_after_1 = helpers.get_map_reduces_stat(router, 'customers') + local diff_1 = map_reduces_after_1 - map_reduces_before + t.assert_equals(diff_1, 0, 'Select request was not a map reduce') + + -- Case: EQ on secondary index, which is not in the sharding + -- index (primary index in the case). + local result, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select( 'customers', {{'==', 'age', 81}}, {bucket_id = 1161, timeout = 1}) + + foo:close() + return result, err + ]]) + t.assert_equals(err, nil) + t.assert_equals(result.rows, { + {4, 1161, 'William', 'White', 81, 'Chicago'}, + }) + + local map_reduces_after_2 = helpers.get_map_reduces_stat(router, 'customers') + local diff_2 = map_reduces_after_2 - map_reduces_after_1 + t.assert_equals(diff_2, 0, 'Select request was not a map reduce') +end + +pgroup.test_select_yield_every_0_readview = function(g) + local resp, err = g.cluster.main_server.net_box:eval([[ + local crud = require('crud') + + foo, err = crud.readview('foo') + + local result, err = foo:select('customers', nil, { yield_every = 0, fullscan = true }) + + foo:close() + return result, err + ]]) + t.assert_equals(resp, nil) + t.assert_str_contains(err.err, "yield_every should be > 0") +end \ No newline at end of file