From 91d7815d64c160dfbc98d981066361d9d9ee3895 Mon Sep 17 00:00:00 2001 From: AnaNek Date: Fri, 8 Apr 2022 17:37:20 +0300 Subject: [PATCH] Add stats collection and pert tests for batch methods Since we have PR #244 it will be nice to collect statistics for batch operations too. To establish the effectiveness of `crud.batch_insert()` method compared to `crud.insert()`, perf tests were added. `crud.insert()` in the loop and `crud.batch_insert()` are compared for different batch sizes. Closes #193 --- crud.lua | 12 +-- crud/stats/operation.lua | 6 ++ test/integration/stats_test.lua | 106 +++++++++++++++++++++++++- test/performance/perf_test.lua | 129 +++++++++++++++++++++++++++++++- 4 files changed, 242 insertions(+), 11 deletions(-) diff --git a/crud.lua b/crud.lua index d450884ac..05d36ec4a 100644 --- a/crud.lua +++ b/crud.lua @@ -36,11 +36,11 @@ crud.insert_object = stats.wrap(insert.object, stats.op.INSERT) -- @refer insert_many.tuples -- @function insert_many -crud.insert_many = insert_many.tuples +crud.insert_many = stats.wrap(insert_many.tuples, stats.op.INSERT_MANY) -- @refer insert_many.objects -- @function insert_object_many -crud.insert_object_many = insert_many.objects +crud.insert_object_many = stats.wrap(insert_many.objects, stats.op.INSERT_MANY) -- @refer get.call -- @function get @@ -56,11 +56,11 @@ crud.replace_object = stats.wrap(replace.object, stats.op.REPLACE) -- @refer replace_many.tuples -- @function replace_many -crud.replace_many = replace_many.tuples +crud.replace_many = stats.wrap(replace_many.tuples, stats.op.REPLACE_MANY) -- @refer replace_many.objects -- @function replace_object_many -crud.replace_object_many = replace_many.objects +crud.replace_object_many = stats.wrap(replace_many.objects, stats.op.REPLACE_MANY) -- @refer update.call -- @function update @@ -72,11 +72,11 @@ crud.upsert = stats.wrap(upsert.tuple, stats.op.UPSERT) -- @refer upsert_many.tuples -- @function upsert_many -crud.upsert_many = upsert_many.tuples +crud.upsert_many = stats.wrap(upsert_many.tuples, stats.op.UPSERT_MANY) -- @refer upsert_many.objects -- @function upsert_object_many -crud.upsert_object_many = upsert_many.objects +crud.upsert_object_many = stats.wrap(upsert_many.objects, stats.op.UPSERT_MANY) -- @refer upsert.object -- @function upsert diff --git a/crud/stats/operation.lua b/crud/stats/operation.lua index a6a9627a4..583cf6725 100644 --- a/crud/stats/operation.lua +++ b/crud/stats/operation.lua @@ -6,12 +6,18 @@ return { -- INSERT identifies both `insert` and `insert_object`. INSERT = 'insert', + -- INSERT_MANY identifies both `insert_many` and `insert_object_many`. + INSERT_MANY = 'insert_many', GET = 'get', -- REPLACE identifies both `replace` and `replace_object`. REPLACE = 'replace', + -- REPLACE_MANY identifies both `replace_many` and `replace_object_many`. + REPLACE_MANY = 'replace_many', UPDATE = 'update', -- UPSERT identifies both `upsert` and `upsert_object`. UPSERT = 'upsert', + -- UPSERT_MANY identifies both `upsert_many` and `upsert_object_many`. + UPSERT_MANY = 'upsert_many', DELETE = 'delete', -- SELECT identifies both `pairs` and `select`. SELECT = 'select', diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua index 02d750d3c..e5008166b 100644 --- a/test/integration/stats_test.lua +++ b/test/integration/stats_test.lua @@ -175,6 +175,28 @@ local simple_operation_cases = { }, op = 'insert', }, + insert_many = { + func = 'crud.insert_many', + args = { + space_name, + { + { 21, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + { 31, box.NULL, 'Oleg', 'Petrov', 25, 'Moscow' }, + } + }, + op = 'insert_many', + }, + insert_object_many = { + func = 'crud.insert_object_many', + args = { + space_name, + { + { id = 22, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + { id = 32, name = 'Oleg', last_name = 'Petrov', age = 25, city = 'Moscow' }, + } + }, + op = 'insert_many', + }, get = { func = 'crud.get', args = { space_name, { 12 } }, @@ -206,6 +228,28 @@ local simple_operation_cases = { }, op = 'replace', }, + replace_many = { + func = 'crud.replace_many', + args = { + space_name, + { + { 21, box.NULL, 'Peter', 'Ivanov', 40, 'Moscow' }, + { 31, box.NULL, 'Ivan', 'Petrov', 35, 'Moscow' }, + } + }, + op = 'replace_many', + }, + replace_object_many = { + func = 'crud.replace_object_many', + args = { + space_name, + { + { id = 22, name = 'Peter', last_name = 'Ivanov', age = 40, city = 'Moscow' }, + { id = 32, name = 'Ivan', last_name = 'Petrov', age = 35, city = 'Moscow' }, + } + }, + op = 'replace_many', + }, update = { prepare = function(g) helpers.insert_objects(g, space_name, {{ @@ -235,6 +279,28 @@ local simple_operation_cases = { }, op = 'upsert', }, + upsert_many = { + func = 'crud.upsert_many', + args = { + space_name, + { + {{ 26, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, {{'+', 'age', 1}}}, + {{ 36, box.NULL, 'Oleg', 'Petrov', 25, 'Moscow' }, {{'+', 'age', 1}}}, + }, + }, + op = 'upsert_many', + }, + upsert_object_many = { + func = 'crud.upsert_object_many', + args = { + space_name, + { + {{ id = 27, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, {{'+', 'age', 1}}}, + {{ id = 37, name = 'Oleg', last_name = 'Petrov', age = 25, city = 'Moscow' }, {{'+', 'age', 1}}}, + }, + }, + op = 'upsert_many', + }, delete = { func = 'crud.delete', args = { space_name, { 12 } }, @@ -277,6 +343,18 @@ local simple_operation_cases = { op = 'insert', expect_error = true, }, + insert_many_error = { + func = 'crud.insert_many', + args = { space_name, {{ 'id' }} }, + op = 'insert_many', + expect_error = true, + }, + insert_object_many_error = { + func = 'crud.insert_object_many', + args = { space_name, {{ 'id' }} }, + op = 'insert_many', + expect_error = true, + }, get_error = { func = 'crud.get', args = { space_name, { 'id' } }, @@ -308,6 +386,18 @@ local simple_operation_cases = { op = 'replace', expect_error = true, }, + replace_many_error = { + func = 'crud.replace_many', + args = { space_name, {{ 'id' }} }, + op = 'replace_many', + expect_error = true, + }, + replace_object_many_error = { + func = 'crud.replace_object_many', + args = { space_name, {{ 'id' }} }, + op = 'replace_many', + expect_error = true, + }, update_error = { func = 'crud.update', args = { space_name, { 'id' }, {{'+', 'age', 1}} }, @@ -326,6 +416,18 @@ local simple_operation_cases = { op = 'upsert', expect_error = true, }, + upsert_many_error = { + func = 'crud.upsert_many', + args = { space_name, { {{ 'id' }, {{'+', 'age', 1}}} }, }, + op = 'upsert_many', + expect_error = true, + }, + upsert_object_many_error = { + func = 'crud.upsert_object_many', + args = { space_name, { {{ 'id' }, {{'+', 'age', 1}}} } }, + op = 'upsert_many', + expect_error = true, + }, delete_error = { func = 'crud.delete', args = { space_name, { 'id' } }, @@ -800,8 +902,8 @@ local function validate_metrics(g, metrics) t.assert_type(stats_sum, 'table', '`tnt_crud_stats` summary metrics found') - local expected_operations = { 'insert', 'get', 'replace', 'update', - 'upsert', 'delete', 'select', 'truncate', 'len', 'count', 'borders' } + local expected_operations = { 'insert', 'insert_many', 'get', 'replace', 'replace_many', 'update', + 'upsert', 'upsert_many', 'delete', 'select', 'truncate', 'len', 'count', 'borders' } if g.params.quantiles == true then t.assert_items_equals(get_unique_label_values(quantile_stats, 'operation'), expected_operations, diff --git a/test/performance/perf_test.lua b/test/performance/perf_test.lua index 610a44cc1..e2c9e1678 100644 --- a/test/performance/perf_test.lua +++ b/test/performance/perf_test.lua @@ -4,6 +4,7 @@ local fiber = require('fiber') local errors = require('errors') local net_box = require('net.box') local log = require('log') +local fun = require('fun') local t = require('luatest') local g = t.group('perf') @@ -98,8 +99,32 @@ local function normalize(s, n) return (' '):rep(n - len) .. s end +local function generate_batch_insert_cases(min_value, denominator, count) + local matrix = {} + local cols_names = {} + + local batch_size = min_value + for _ = 1, count do + local col_name = tostring(batch_size) + table.insert(cols_names, col_name) + matrix[col_name] = { column_name = col_name, arg = batch_size } + batch_size = batch_size * denominator + end + + return { + matrix = matrix, + cols_names = cols_names, + } +end + +local min_batch_size = 1 +local denominator_batch_size = 10 +local count_batch_cases = 5 +local batch_insert_cases = generate_batch_insert_cases(min_batch_size, denominator_batch_size, count_batch_cases) + local row_name = { insert = 'insert', + insert_many = 'insert_many', select_pk = 'select by pk', select_gt_pk = 'select gt by pk (limit 10)', select_secondary_eq = 'select eq by secondary (limit 10)', @@ -117,6 +142,12 @@ local column_name = { metrics_quantile_stats = 'crud (metrics stats, with quantiles)', } +-- insert column names for insert_many/insert comparison cases +fun.reduce( + function(list, value) list[value] = value return list end, + column_name, pairs(batch_insert_cases.cols_names) +) + local function visualize_section(total_report, name, comment, section, params) local report_str = ('== %s ==\n(%s)\n\n'):format(name, comment or '') @@ -242,6 +273,7 @@ g.after_all(function(g) row_name.select_secondary_eq, row_name.select_secondary_sharded, row_name.insert, + row_name.insert_many, } }) @@ -260,6 +292,15 @@ g.after_all(function(g) row_name.insert, } }) + + visualize_report(g.total_report, 'BATCH COMPARISON PERFORMANCE REPORT', { + columns = batch_insert_cases.cols_names, + + rows = { + row_name.insert, + row_name.insert_many, + }, + }) end) local function generate_customer() @@ -488,6 +529,18 @@ local insert_params = function() return { 'customers', generate_customer() } end +local batch_insert_params = function(count) + local batch = {} + + count = count or 1 + + for _ = 1, count do + table.insert(batch, generate_customer()) + end + + return { 'customers', batch } +end + local select_params_pk_eq = function() return { 'customers', {{'==', 'id', gen() % 10000}} } end @@ -606,6 +659,12 @@ local pairs_perf = { connection_count = 10, } +local batch_insert_comparison_perf = { + timeout = 30, + fiber_count = 1, + connection_count = 1, +} + local cases = { vshard_insert = { prepare = vshard_prepare, @@ -640,6 +699,29 @@ local cases = { row_name = row_name.insert, }, + crud_insert_many = { + call = 'crud.insert_many', + params = batch_insert_params, + matrix = stats_cases, + integration_params = integration_params, + perf_params = insert_perf, + row_name = row_name.insert_many, + }, + + crud_insert_many_without_stats_wrapper = { + prepare = function(g) + g.router:eval([[ + rawset(_G, '_plain_insert_many', require('crud.insert_many').tuples) + ]]) + end, + call = '_plain_insert_many', + params = batch_insert_params, + matrix = { [''] = { column_name = column_name.without_stats_wrapper } }, + integration_params = integration_params, + perf_params = batch_insert_comparison_perf, + row_name = row_name.insert_many, + }, + vshard_select_pk_eq = { prepare = function(g) select_prepare(g) @@ -876,14 +958,55 @@ local cases = { perf_params = pairs_perf, row_name = row_name.pairs_gt, }, + + crud_insert_loop = { + prepare = function(g) + g.router:eval([[ + _insert_loop = function(space_name, tuples) + local results + local errors + + for _, tuple in ipairs(tuples) do + local res, err = crud.insert(space_name, tuple) + if res ~= nil then + results = results or {} + table.insert(results, res) + end + + if err ~= nil then + errors = errors or {} + table.insert(errors, err) + end + end + + return results, errors + end + ]]) + end, + call = '_insert_loop', + params = batch_insert_params, + matrix = batch_insert_cases.matrix, + integration_params = integration_params, + perf_params = batch_insert_comparison_perf, + row_name = row_name.insert, + }, + + crud_insert_many_different_batch_size = { + call = 'crud.insert_many', + params = batch_insert_params, + matrix = batch_insert_cases.matrix, + integration_params = integration_params, + perf_params = batch_insert_comparison_perf, + row_name = row_name.insert_many, + }, } -local function generator_f(conn, call, params, report, timeout) +local function generator_f(conn, call, params, report, timeout, arg) local start = clock.monotonic() while (clock.monotonic() - start) < timeout do local call_start = clock.monotonic() - local ok, res, err = pcall(conn.call, conn, call, params()) + local ok, res, err = pcall(conn.call, conn, call, params(arg)) local call_time = clock.monotonic() - call_start if not ok then @@ -947,7 +1070,7 @@ for name, case in pairs(cases) do for id = 1, params.fiber_count do local conn_id = id % params.connection_count + 1 local conn = connections[conn_id] - local f = fiber.new(generator_f, conn, case.call, case.params, report, params.timeout) + local f = fiber.new(generator_f, conn, case.call, case.params, report, params.timeout, subcase.arg) f:set_joinable(true) table.insert(fibers, f) end