-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Batch upsert is mostly used for operation with one bucket / one Tarantool node in a transaction. In this case batch replace is more efficient then replacing tuple-by-tuple. Right now CRUD cannot provide batch replace with full consistency. CRUD offers batch upsert with partial consistency. That means that full consistency can be provided only on single replicaset using `box` transactions. Part of #193
- Loading branch information
Showing
6 changed files
with
2,135 additions
and
140 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,246 @@ | ||
| local checks = require('checks') | ||
| local errors = require('errors') | ||
| local vshard = require('vshard') | ||
|
|
||
| local call = require('crud.common.call') | ||
| local const = require('crud.common.const') | ||
| local utils = require('crud.common.utils') | ||
| local sharding = require('crud.common.sharding') | ||
| local dev_checks = require('crud.common.dev_checks') | ||
| local schema = require('crud.common.schema') | ||
|
|
||
| local BatchInsertIterator = require('crud.common.map_call_cases.batch_insert_iter') | ||
| local BatchPostprocessor = require('crud.common.map_call_cases.batch_postprocessor') | ||
|
|
||
| local BatchReplaceError = errors.new_class('BatchReplaceError', {capture_stack = false}) | ||
|
|
||
| local batch_replace = {} | ||
|
|
||
| local BATCH_REPLACE_FUNC_NAME = '_crud.batch_replace_on_storage' | ||
|
|
||
| local function batch_replace_on_storage(space_name, tuples, opts) | ||
| dev_checks('string', 'table', { | ||
| add_space_schema_hash = '?boolean', | ||
| fields = '?table', | ||
| stop_on_error = '?boolean', | ||
| rollback_on_error = '?boolean', | ||
| sharding_key_hash = '?number', | ||
| sharding_func_hash = '?number', | ||
| skip_sharding_hash_check = '?boolean', | ||
| }) | ||
|
|
||
| opts = opts or {} | ||
|
|
||
| local space = box.space[space_name] | ||
| if space == nil then | ||
| return nil, {BatchReplaceError:new("Space %q doesn't exist", space_name)} | ||
| end | ||
|
|
||
| local _, err = sharding.check_sharding_hash(space_name, | ||
| opts.sharding_func_hash, | ||
| opts.sharding_key_hash, | ||
| opts.skip_sharding_hash_check) | ||
|
|
||
| if err ~= nil then | ||
| return nil, {err} | ||
| end | ||
|
|
||
| local inserted_tuples = {} | ||
| local errs = {} | ||
|
|
||
| box.begin() | ||
| for _, tuple in ipairs(tuples) do | ||
| -- add_space_schema_hash is true only in case of replace_object_many | ||
| -- the only one case when reloading schema can avoid replace error | ||
| -- is flattening object on router | ||
| local insert_result = schema.wrap_box_space_func_result(space, 'replace', {tuple}, { | ||
| add_space_schema_hash = opts.add_space_schema_hash, | ||
| field_names = opts.fields, | ||
| }) | ||
|
|
||
| if insert_result.err ~= nil then | ||
| local err = { | ||
| err = insert_result.err, | ||
| tuple = tuple, | ||
| } | ||
|
|
||
| if opts.stop_on_error == true then | ||
| if opts.rollback_on_error == true then | ||
| box.rollback() | ||
| return nil, {err} | ||
| end | ||
|
|
||
| box.commit() | ||
|
|
||
| return inserted_tuples, {err} | ||
| end | ||
|
|
||
| table.insert(errs, err) | ||
| end | ||
|
|
||
| table.insert(inserted_tuples, insert_result.res) | ||
| end | ||
|
|
||
| if next(errs) ~= nil then | ||
| if opts.rollback_on_error == true then | ||
| box.rollback() | ||
| return nil, errs | ||
| end | ||
|
|
||
| box.commit() | ||
|
|
||
| return inserted_tuples, errs | ||
| end | ||
|
|
||
| box.commit() | ||
|
|
||
| return inserted_tuples | ||
| end | ||
|
|
||
| function batch_replace.init() | ||
| _G._crud.batch_replace_on_storage = batch_replace_on_storage | ||
| end | ||
|
|
||
| -- returns result, err, need_reload | ||
| -- need_reload indicates if reloading schema could help | ||
| -- see crud.common.schema.wrap_func_reload() | ||
| local function call_batch_replace_on_router(space_name, original_tuples, opts) | ||
| dev_checks('string', 'table', { | ||
| timeout = '?number', | ||
| fields = '?table', | ||
| add_space_schema_hash = '?boolean', | ||
| stop_on_error = '?boolean', | ||
| rollback_on_error = '?boolean', | ||
| }) | ||
|
|
||
| opts = opts or {} | ||
|
|
||
| local space = utils.get_space(space_name, vshard.router.routeall()) | ||
| if space == nil then | ||
| return nil, {BatchReplaceError:new("Space %q doesn't exist", space_name)}, true | ||
| end | ||
|
|
||
| local tuples = table.deepcopy(original_tuples) | ||
|
|
||
| local batch_replace_on_storage_opts = { | ||
| add_space_schema_hash = opts.add_space_schema_hash, | ||
| fields = opts.fields, | ||
| stop_on_error = opts.stop_on_error, | ||
| rollback_on_error = opts.rollback_on_error, | ||
| } | ||
|
|
||
| local iter, err = BatchInsertIterator:new({ | ||
| tuples = tuples, | ||
| space = space, | ||
| execute_on_storage_opts = batch_replace_on_storage_opts, | ||
| }) | ||
| if err ~= nil then | ||
| return nil, {err}, const.NEED_SCHEMA_RELOAD | ||
| end | ||
|
|
||
| local postprocessor = BatchPostprocessor:new() | ||
|
|
||
| local rows, errs = call.map(BATCH_REPLACE_FUNC_NAME, nil, { | ||
| timeout = opts.timeout, | ||
| mode = 'write', | ||
| iter = iter, | ||
| postprocessor = postprocessor, | ||
| }) | ||
|
|
||
| if next(rows) == nil then | ||
| return nil, errs | ||
| end | ||
|
|
||
| local res, err = utils.format_result(rows, space, opts.fields) | ||
| if err ~= nil then | ||
| return nil, {err} | ||
| end | ||
|
|
||
| return res, errs | ||
| end | ||
|
|
||
| --- Batch replace tuples to the specified space | ||
| -- | ||
| -- @function tuples_batch | ||
| -- | ||
| -- @param string space_name | ||
| -- A space name | ||
| -- | ||
| -- @param table tuples | ||
| -- Tuples | ||
| -- | ||
| -- @tparam ?table opts | ||
| -- Options of batch_replace.tuples_batch | ||
| -- | ||
| -- @return[1] tuples | ||
| -- @treturn[2] nil | ||
| -- @treturn[2] table of tables Error description | ||
|
|
||
| function batch_replace.tuples_batch(space_name, tuples, opts) | ||
| checks('string', 'table', { | ||
| timeout = '?number', | ||
| fields = '?table', | ||
| add_space_schema_hash = '?boolean', | ||
| stop_on_error = '?boolean', | ||
| rollback_on_error = '?boolean', | ||
| }) | ||
|
|
||
| return schema.wrap_func_reload(call_batch_replace_on_router, space_name, tuples, opts) | ||
| end | ||
|
|
||
| --- Batch replace objects to the specified space | ||
| -- | ||
| -- @function objects_batch | ||
| -- | ||
| -- @param string space_name | ||
| -- A space name | ||
| -- | ||
| -- @param table objs | ||
| -- Objects | ||
| -- | ||
| -- @tparam ?table opts | ||
| -- Options of batch_insert.tuples_batch | ||
| -- | ||
| -- @return[1] objects | ||
| -- @treturn[2] nil | ||
| -- @treturn[2] table of tables Error description | ||
|
|
||
| function batch_replace.objects_batch(space_name, objs, opts) | ||
| checks('string', 'table', { | ||
| timeout = '?number', | ||
| fields = '?table', | ||
| stop_on_error = '?boolean', | ||
| rollback_on_error = '?boolean', | ||
| }) | ||
|
|
||
| -- insert can fail if router uses outdated schema to flatten object | ||
| opts = utils.merge_options(opts, {add_space_schema_hash = true}) | ||
|
|
||
| local tuples = {} | ||
| local errs = {} | ||
|
|
||
| for _, obj in ipairs(objs) do | ||
|
|
||
| local tuple, err = utils.flatten_obj_reload(space_name, obj) | ||
| if err ~= nil then | ||
| local err_obj = BatchReplaceError:new("Failed to flatten object: %s", err) | ||
| err_obj.tuple = obj | ||
|
|
||
| if opts.stop_on_error == true then | ||
| return nil, {err_obj} | ||
| end | ||
|
|
||
| table.insert(errs, err_obj) | ||
| end | ||
|
|
||
| table.insert(tuples, tuple) | ||
| end | ||
|
|
||
| if next(errs) ~= nil then | ||
| return nil, errs | ||
| end | ||
|
|
||
| return batch_replace.tuples_batch(space_name, tuples, opts) | ||
| end | ||
|
|
||
| return batch_replace |
Oops, something went wrong.