From 5685009104b293be228669d6d628e0eb7cbd4f4b Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Mon, 13 Dec 2021 15:20:23 +0300 Subject: [PATCH] Add statistics for CRUD operations on router Add statistics module for collecting metrics of CRUD operations on router. Wrap all CRUD operation calls in statistics collector. Statistics must be enabled manually, and they can be disabled, restarted or re-enabled later. `crud.stats()` returns --- - spaces: my_space: insert: ok: latency: 0.002 count: 19800 time: 39.6 error: latency: 0.000001 count: 4 time: 0.000004 space_not_found: 0 ... `spaces` section contains statistics for each observed space. If operation has never been called for space, corresponding field will be empty. If no operations has been called for a space, it will not be represented. Operations called for spaces not found both on storages and in statistics registry (including cases when crud can't verify space existence) will increment `space_not_found` counter. Possible statistics operation labels are `insert` (for `insert` and `insert_object` calls), `get`, `replace` (for `replace` and `replace_object` calls), `update`, `upsert` (for `upsert` and `upsert_object` calls), `delete`, `select` (for `select` and `pairs` calls), `truncate`, `len` and `borders` (for `min` and `max` calls). Each operation section contains of different collectors for success calls and error (both error throw and `nil, err`) returns. `count` is total requests count since instance start or stats restart. `latency` is average time of requests execution, `time` is total time of requests execution. Part of #224 --- CHANGELOG.md | 1 + README.md | 71 +++++ crud.lua | 47 ++-- crud/stats/local_registry.lua | 124 +++++++++ crud/stats/module.lua | 211 +++++++++++++++ crud/stats/operation.lua | 17 ++ crud/stats/registry_common.lua | 56 ++++ test/helper.lua | 69 +++++ test/integration/stats_test.lua | 447 ++++++++++++++++++++++++++++++++ test/unit/stats_test.lua | 443 +++++++++++++++++++++++++++++++ 10 files changed, 1471 insertions(+), 15 deletions(-) create mode 100644 crud/stats/local_registry.lua create mode 100644 crud/stats/module.lua create mode 100644 crud/stats/operation.lua create mode 100644 crud/stats/registry_common.lua create mode 100644 test/integration/stats_test.lua create mode 100644 test/unit/stats_test.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f606f62..0a3d9325 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +* Statistics for CRUD operations on router (#224). ### Changed diff --git a/README.md b/README.md index c62a5412..3fb06975 100644 --- a/README.md +++ b/README.md @@ -586,6 +586,77 @@ crud.len('customers') ... ``` +### Statistics + +`crud` routers can provide statistics on called operations. +```lua +-- Enable statistics collect. +crud.enable_stats() + +-- Returns table with statistics information. +crud.stats() + +-- Returns table with statistics information for specific space. +crud.stats('my_space') + +-- Disables statistics collect and destroys all collectors. +crud.disable_stats() + +-- Destroys all statistics collectors and creates them again. +crud.reset_stats() +``` + +Format is as follows. +```lua +crud.stats() +--- +- spaces: + my_space: + insert: + ok: + latency: 0.002 + count: 19800 + time: 39.6 + error: + latency: 0.000001 + count: 4 + time: 0.000004 + space_not_found: 0 +... +crud.stats('my_space') +--- +- insert: + ok: + latency: 0.002 + count: 19800 + time: 39.6 + error: + latency: 0.000001 + count: 4 + time: 0.000004 +... +``` +`spaces` section contains statistics for each observed space. +If operation has never been called for space, corresponding +field will be empty. If no operations has been called for a +space, it will not be represented. Operations called for +spaces not found both on storages and in statistics registry +(including cases when crud can't verify space existence) +will increment `space_not_found` counter. + +Possible statistics operation labels are +`insert` (for `insert` and `insert_object` calls), +`get`, `replace` (for `replace` and `replace_object` calls), `update`, +`upsert` (for `upsert` and `upsert_object` calls), `delete`, +`select` (for `select` and `pairs` calls), `truncate`, `len` and +`borders` (for `min` and `max` calls). + +Each operation section contains of different collectors +for success calls and error (both error throw and `nil, err`) +returns. `count` is total requests count since instance start +or stats restart. `latency` is average time of requests execution, +`time` is total time of requests execution. + ## Cartridge roles `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the diff --git a/crud.lua b/crud.lua index 2777013e..cef477e3 100644 --- a/crud.lua +++ b/crud.lua @@ -14,6 +14,7 @@ local len = require('crud.len') local borders = require('crud.borders') local sharding_key = require('crud.common.sharding_key') local utils = require('crud.common.utils') +local stats = require('crud.stats.module') local crud = {} @@ -22,47 +23,47 @@ local crud = {} -- @refer insert.tuple -- @function insert -crud.insert = insert.tuple +crud.insert = stats.wrap(insert.tuple, stats.op.INSERT) -- @refer insert.object -- @function insert_object -crud.insert_object = insert.object +crud.insert_object = stats.wrap(insert.object, stats.op.INSERT) -- @refer get.call -- @function get -crud.get = get.call +crud.get = stats.wrap(get.call, stats.op.GET) -- @refer replace.tuple -- @function replace -crud.replace = replace.tuple +crud.replace = stats.wrap(replace.tuple, stats.op.REPLACE) -- @refer replace.object -- @function replace_object -crud.replace_object = replace.object +crud.replace_object = stats.wrap(replace.object, stats.op.REPLACE) -- @refer update.call -- @function update -crud.update = update.call +crud.update = stats.wrap(update.call, stats.op.UPDATE) -- @refer upsert.tuple -- @function upsert -crud.upsert = upsert.tuple +crud.upsert = stats.wrap(upsert.tuple, stats.op.UPSERT) -- @refer upsert.object -- @function upsert -crud.upsert_object = upsert.object +crud.upsert_object = stats.wrap(upsert.object, stats.op.UPSERT) -- @refer delete.call -- @function delete -crud.delete = delete.call +crud.delete = stats.wrap(delete.call, stats.op.DELETE) -- @refer select.call -- @function select -crud.select = select.call +crud.select = stats.wrap(select.call, stats.op.SELECT) -- @refer select.pairs -- @function pairs -crud.pairs = select.pairs +crud.pairs = stats.wrap(select.pairs, stats.op.SELECT, { pairs = true }) -- @refer utils.unflatten_rows -- @function unflatten_rows @@ -70,19 +71,19 @@ crud.unflatten_rows = utils.unflatten_rows -- @refer truncate.call -- @function truncate -crud.truncate = truncate.call +crud.truncate = stats.wrap(truncate.call, stats.op.TRUNCATE) -- @refer len.call -- @function len -crud.len = len.call +crud.len = stats.wrap(len.call, stats.op.LEN) -- @refer borders.min -- @function min -crud.min = borders.min +crud.min = stats.wrap(borders.min, stats.op.BORDERS) -- @refer borders.max -- @function max -crud.max = borders.max +crud.max = stats.wrap(borders.max, stats.op.BORDERS) -- @refer utils.cut_rows -- @function cut_rows @@ -92,6 +93,22 @@ crud.cut_rows = utils.cut_rows -- @function cut_objects crud.cut_objects = utils.cut_objects +-- @refer stats.enable +-- @function enable_stats +crud.enable_stats = stats.enable + +-- @refer stats.get +-- @function stats +crud.stats = stats.get + +-- @refer stats.disable +-- @function disable_stats +crud.disable_stats = stats.disable + +-- @refer stats.reset +-- @function reset_stats +crud.reset_stats = stats.reset + --- Initializes crud on node -- -- Exports all functions that are used for calls diff --git a/crud/stats/local_registry.lua b/crud/stats/local_registry.lua new file mode 100644 index 00000000..26b1d6fb --- /dev/null +++ b/crud/stats/local_registry.lua @@ -0,0 +1,124 @@ +local dev_checks = require('crud.common.dev_checks') +local registry_common = require('crud.stats.registry_common') + +local registry = {} +local internal_registry = {} + +--- Initialize local metrics registry +-- +-- Registries are not meant to used explicitly +-- by users, init is not guaranteed to be idempotent. +-- +-- @function init +-- +-- @treturn boolean Returns true. +-- +function registry.init() + internal_registry.spaces = {} + internal_registry.space_not_found = 0 + + return true +end + +--- Destroy local metrics registry +-- +-- Registries are not meant to used explicitly +-- by users, destroy is not guaranteed to be idempotent. +-- +-- @function destroy +-- +-- @treturn boolean Returns true. +-- +function registry.destroy() + internal_registry = {} + + return true +end + +--- Get copy of local metrics registry +-- +-- Registries are not meant to used explicitly +-- by users, get is not guaranteed to work without init. +-- +-- @function get +-- +-- @tparam string space_name +-- (Optional) If specified, returns table with statistics +-- of operations on table, separated by operation type and +-- execution status. If there wasn't any requests for table, +-- returns {}. In not specified, returns table with statistics +-- about all existing spaces and count of calls to spaces +-- that wasn't found. +-- +-- @treturn table Returns copy of metrics registry (or registry section). +-- +function registry.get(space_name) + dev_checks('?string') + + if space_name ~= nil then + return table.deepcopy(internal_registry.spaces[space_name]) or {} + end + + return table.deepcopy(internal_registry) +end + +--- Check if space statistics are present in registry +-- +-- @function is_unknown_space +-- +-- @tparam string space_name +-- Name of space. +-- +-- @treturn boolean True, if space stats found. False otherwise. +-- +function registry.is_unknown_space(space_name) + dev_checks('string') + + return internal_registry.spaces[space_name] == nil +end + +--- Increase requests count and update latency info +-- +-- @function observe +-- +-- @tparam string space_name +-- Name of space. +-- +-- @tparam number latency +-- Time of call execution. +-- +-- @tparam string op +-- Label of registry collectors. +-- Use `require('crud.common.const').OP` to pick one. +-- +-- @tparam string success +-- 'ok' if no errors on execution, 'error' otherwise. +-- +-- @treturn boolean Returns true. +-- +function registry.observe(latency, space_name, op, status) + dev_checks('number', 'string', 'string', 'string') + + registry_common.init_collectors_if_required(internal_registry.spaces, space_name, op) + local collectors = internal_registry.spaces[space_name][op][status] + + collectors.count = collectors.count + 1 + collectors.time = collectors.time + latency + collectors.latency = collectors.time / collectors.count + + return true +end + +--- Increase count of "space not found" collector by one +-- +-- @function observe_space_not_found +-- +-- @treturn boolean Returns true. +-- +function registry.observe_space_not_found() + internal_registry.space_not_found = internal_registry.space_not_found + 1 + + return true +end + +return registry diff --git a/crud/stats/module.lua b/crud/stats/module.lua new file mode 100644 index 00000000..e7b37741 --- /dev/null +++ b/crud/stats/module.lua @@ -0,0 +1,211 @@ +local clock = require('clock') +local checks = require('checks') +local errors = require('errors') +local vshard = require('vshard') + +local dev_checks = require('crud.common.dev_checks') +local utils = require('crud.common.utils') +local op_module = require('crud.stats.operation') +local registry = require('crud.stats.local_registry') + +local StatsError = errors.new_class('StatsError', {capture_stack = false}) + +local stats = {} +local is_enabled = false + +--- Initializes statistics registry, enables callbacks and wrappers +-- +-- If already enabled, do nothing. +-- +-- @function enable +-- +-- @treturn boolean Returns true. +-- +function stats.enable() + if is_enabled then + return true + end + + StatsError:assert( + rawget(_G, 'crud') ~= nil, + "Can be enabled only on crud router" + ) + + registry.init() + is_enabled = true + + return true +end + +--- Resets statistics registry +-- +-- After reset collectors set is the same as right +-- after first stats.enable(). +-- +-- @function reset +-- +-- @treturn boolean Returns true. +-- +function stats.reset() + if not is_enabled then + return true + end + + registry.destroy() + registry.init() + + return true +end + +--- Destroys statistics registry and disable callbacks +-- +-- If already disabled, do nothing. +-- +-- @function disable +-- +-- @treturn boolean Returns true. +-- +function stats.disable() + if not is_enabled then + return true + end + + registry.destroy() + is_enabled = false + + return true +end + +--- Get statistics on CRUD operations +-- +-- @function get +-- +-- @tparam string space_name +-- (Optional) If specified, returns table with statistics +-- of operations on space, separated by operation type and +-- execution status. If there wasn't any requests of "op" type +-- for space, there won't be corresponding collectors. +-- If not specified, returns table with statistics +-- about all observed spaces and count of calls to spaces +-- that wasn't found. +-- +-- @treturn table Statistics on CRUD operations. +-- If statistics disabled, returns {}. +-- +function stats.get(space_name) + checks('?string') + + return registry.get(space_name) +end + +local function wrap_tail(space_name, op, opts, start_time, call_status, ...) + local finish_time = clock.monotonic() + local latency = finish_time - start_time + + local err = nil + local status = 'ok' + if call_status == false then + status = 'error' + err = select(1, ...) + end + + -- If not `pairs` call, return values `nil, err` + -- treated as error case. + local second_return_val = select(2, ...) + if opts.pairs == false and second_return_val ~= nil then + status = 'error' + err = second_return_val + end + + -- If space not exists, do not build a separate collector for it. + -- Call request for non-existing space will always result in error. + -- The resulting overhead is insignificant for existing spaces: + -- at worst it would be a single excessive check for an instance lifetime. + -- If we can't verify space existence because of network errors, + -- it is treated as unknown as well. + if status == 'error' and registry.is_unknown_space(space_name) then + if type(err) == 'table' and type(err.err) == 'string' then + local space_not_found_msg = utils.space_doesnt_exist_msg(space_name) + if string.find(err.err, space_not_found_msg) ~= nil then + registry.observe_space_not_found() + goto return_values + end + end + + -- We can't rely only on parsing error value because space existence + -- is not always the first check in request validation. + -- Check explicitly if space do not exist. + local space = utils.get_space(space_name, vshard.router.routeall()) + if space == nil then + registry.observe_space_not_found() + goto return_values + end + end + + registry.observe(latency, space_name, op, status) + + :: return_values :: + + if call_status == false then + error((...), 2) + end + + return ... +end + +--- Wrap CRUD operation call to collect statistics +-- +-- Approach based on `box.atomic()`: +-- https://github.com/tarantool/tarantool/blob/b9f7204b5e0d10b443c6f198e9f7f04e0d16a867/src/box/lua/schema.lua#L369 +-- +-- @function wrap +-- +-- @tparam function func +-- Function to wrap. First argument is expected to +-- be a space name string. If statistics enabled, +-- errors are caught and thrown again. +-- +-- @tparam string op +-- Label of registry collectors. +-- Use `require('crud.stats.module').op` to pick one. +-- +-- @tparam table opts +-- +-- @tfield boolean pairs +-- (Optional, default: false) If false, second return value +-- of wrapped function is treated as error (`nil, err` case). +-- Since pairs calls return three arguments as generator +-- and throw errors if needed, use { pairs = true } to +-- wrap them. +-- +-- @return First two arguments of wrapped function output. +-- +function stats.wrap(func, op, opts) + dev_checks('function', 'string', { pairs = '?boolean' }) + + return function(...) + if not is_enabled then + return func(...) + end + + if opts == nil then opts = {} end + if opts.pairs == nil then opts.pairs = false end + + local space_name = select(1, ...) + + local start_time = clock.monotonic() + + return wrap_tail( + space_name, op, opts, start_time, + pcall(func, ...) + ) + end +end + +--- Table with CRUD operation lables +-- +-- @table label +-- +stats.op = op_module + +return stats diff --git a/crud/stats/operation.lua b/crud/stats/operation.lua new file mode 100644 index 00000000..ee1f9826 --- /dev/null +++ b/crud/stats/operation.lua @@ -0,0 +1,17 @@ +return { + -- INSERT identifies both `insert` and `insert_object`. + INSERT = 'insert', + GET = 'get', + -- REPLACE identifies both `replace` and `replace_object`. + REPLACE = 'replace', + UPDATE = 'update', + -- UPSERT identifies both `upsert` and `upsert_object`. + UPSERT = 'upsert', + DELETE = 'delete', + -- SELECT identifies both `pairs` and `select`. + SELECT = 'select', + TRUNCATE = 'truncate', + LEN = 'len', + -- BORDERS identifies both `min` and `max`. + BORDERS = 'borders', +} diff --git a/crud/stats/registry_common.lua b/crud/stats/registry_common.lua new file mode 100644 index 00000000..084bebb9 --- /dev/null +++ b/crud/stats/registry_common.lua @@ -0,0 +1,56 @@ +local dev_checks = require('crud.common.dev_checks') + +local registry_common = {} + +--- Build collectors for local registry +-- +-- @function build_collectors +-- +-- @treturn table Returns collectors for success and error requests. +-- Collectors store 'count', 'latency' and 'time' values. +-- +function registry_common.build_collectors() + local collectors = { + ok = { + count = 0, + latency = 0, + time = 0, + }, + error = { + count = 0, + latency = 0, + time = 0, + }, + } + + return collectors +end + +--- Initialize all statistic collectors for a space operation +-- +-- @function init_collectors_if_required +-- +-- @tparam table spaces +-- `spaces` section of registry. +-- +-- @tparam string space_name +-- Name of space. +-- +-- @tparam string op +-- Label of registry collectors. +-- Use `require('crud.stats.module').op` to pick one. +-- +function registry_common.init_collectors_if_required(spaces, space_name, op) + dev_checks('table', 'string', 'string') + + if spaces[space_name] == nil then + spaces[space_name] = {} + end + + local space_collectors = spaces[space_name] + if space_collectors[op] == nil then + space_collectors[op] = registry_common.build_collectors() + end +end + +return registry_common diff --git a/test/helper.lua b/test/helper.lua index 913aa56c..e78819de 100644 --- a/test/helper.lua +++ b/test/helper.lua @@ -332,4 +332,73 @@ function helpers.update_cache(cluster, space_name) ]], {space_name}) end +function helpers.simple_functions_params() + return { + sleep_time = 0.01, + error = { err = 'err' }, + error_msg = 'throw me', + } +end + +function helpers.prepare_simple_functions(router) + local params = helpers.simple_functions_params() + + local _, err = router:eval([[ + local clock = require('clock') + local fiber = require('fiber') + + local params = ... + local sleep_time = params.sleep_time + local error_table = params.error + local error_msg = params.error_msg + + -- Using `fiber.sleep(time)` between two `clock.monotonic()` + -- may return diff less than `time`. + local function sleep_for(time) + local start = clock.monotonic() + while (clock.monotonic() - start) < time do + fiber.sleep(time / 10) + end + end + + return_true = function(space_name) -- luacheck: ignore + sleep_for(sleep_time) + return true + end + + return_err = function(space_name) -- luacheck: ignore + sleep_for(sleep_time) + return nil, error_table + end + + pairs_ok = function(space_name) -- luacheck: ignore + sleep_for(sleep_time) + return pairs({}) + end + + throws_error = function() + sleep_for(sleep_time) + error(error_msg) + end + ]], { params }) + + t.assert_equals(err, nil) +end + +function helpers.is_space_exist(router, space_name) + local res, err = router:eval([[ + local vshard = require('vshard') + local utils = require('crud.common.utils') + + local space, err = utils.get_space(..., vshard.router.routeall()) + if err ~= nil then + return nil, err + end + return space ~= nil + ]], { space_name }) + + t.assert_equals(err, nil) + return res +end + return helpers diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua new file mode 100644 index 00000000..52847429 --- /dev/null +++ b/test/integration/stats_test.lua @@ -0,0 +1,447 @@ +local fio = require('fio') +local clock = require('clock') +local t = require('luatest') + +local stats_registry_common = require('crud.stats.registry_common') + +local g = t.group('stats_integration') +local helpers = require('test.helper') + +local space_name = 'customers' +local unknown_space_name = 'non_existing_space' + +g.before_all(function(g) + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_select'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + g.cluster:start() + g.router = g.cluster:server('router').net_box + + helpers.prepare_simple_functions(g.router) + g.router:eval("crud = require('crud')") + g.router:eval("crud.enable_stats()") + + t.assert_equals(helpers.is_space_exist(g.router, space_name), true) + t.assert_equals(helpers.is_space_exist(g.router, unknown_space_name), false) +end) + +g.after_all(function(g) + helpers.stop_cluster(g.cluster) +end) + +g.before_each(function(g) + helpers.truncate_space_on_cluster(g.cluster, space_name) +end) + +function g:get_stats(space_name) + return self.router:eval("return crud.stats(...)", { space_name }) +end + +-- If there weren't any operations, space stats is {}. +-- To compute stats diff, this helper return real stats +-- if they're already present or default stats if +-- this operation of space hasn't been observed yet. +local function get_before_stats(space_stats, op) + if space_stats[op] ~= nil then + return space_stats[op] + else + return stats_registry_common.build_collectors(op) + end +end + +local eval = { + pairs = [[ + local space_name = select(1, ...) + local conditions = select(2, ...) + + local result = {} + for _, v in crud.pairs(space_name, conditions) do + table.insert(result, v) + end + + return result + ]], + + pairs_pcall = [[ + local space_name = select(1, ...) + local conditions = select(2, ...) + + local _, err = pcall(crud.pairs, space_name, conditions) + + return nil, tostring(err) + ]], +} + +-- Call some operations on existing +-- spaces and ensure statistics is updated. +local simple_operation_cases = { + insert = { + func = 'crud.insert', + args = { + space_name, + { 12, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + }, + op = 'insert', + }, + insert_object = { + func = 'crud.insert_object', + args = { + space_name, + { id = 13, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + }, + op = 'insert', + }, + get = { + func = 'crud.get', + args = { space_name, { 12 } }, + op = 'get', + }, + select = { + func = 'crud.select', + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'select', + }, + pairs = { + eval = eval.pairs, + args = { space_name, {{ '==', 'id_index', 3 }} }, + op = 'select', + }, + replace = { + func = 'crud.replace', + args = { + space_name, + { 12, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + }, + op = 'replace', + }, + replace_object = { + func = 'crud.replace_object', + args = { + space_name, + { id = 12, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + }, + op = 'replace', + }, + update = { + prepare = function(g) + helpers.insert_objects(g, space_name, {{ + id = 15, name = 'Ivan', last_name = 'Ivanov', + age = 20, city = 'Moscow' + }}) + end, + func = 'crud.update', + args = { space_name, 12, {{'+', 'age', 10}} }, + op = 'update', + }, + upsert = { + func = 'crud.upsert', + args = { + space_name, + { 16, box.NULL, 'Ivan', 'Ivanov', 20, 'Moscow' }, + {{'+', 'age', 1}}, + }, + op = 'upsert', + }, + upsert_object = { + func = 'crud.upsert_object', + args = { + space_name, + { id = 17, name = 'Ivan', last_name = 'Ivanov', age = 20, city = 'Moscow' }, + {{'+', 'age', 1}} + }, + op = 'upsert', + }, + delete = { + func = 'crud.delete', + args = { space_name, { 12 } }, + op = 'delete', + }, + truncate = { + func = 'crud.truncate', + args = { space_name }, + op = 'truncate', + }, + len = { + func = 'crud.len', + args = { space_name }, + op = 'len', + }, + min = { + func = 'crud.min', + args = { space_name }, + op = 'borders', + }, + max = { + func = 'crud.max', + args = { space_name }, + op = 'borders', + }, + insert_error = { + func = 'crud.insert', + args = { space_name, { 'id' } }, + op = 'insert', + expect_error = true, + }, + insert_object_error = { + func = 'crud.insert_object', + args = { space_name, { 'id' } }, + op = 'insert', + expect_error = true, + }, + get_error = { + func = 'crud.get', + args = { space_name, { 'id' } }, + op = 'get', + expect_error = true, + }, + select_error = { + func = 'crud.select', + args = { space_name, {{ '==', 'id_index', 'sdf' }} }, + op = 'select', + expect_error = true, + }, + pairs_error = { + eval = eval.pairs, + args = { space_name, {{ '%=', 'id_index', 'sdf' }} }, + op = 'select', + expect_error = true, + pcall = true, + }, + replace_error = { + func = 'crud.replace', + args = { space_name, { 'id' } }, + op = 'replace', + expect_error = true, + }, + replace_object_error = { + func = 'crud.replace_object', + args = { space_name, { 'id' } }, + op = 'replace', + expect_error = true, + }, + update_error = { + func = 'crud.update', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'update', + expect_error = true, + }, + upsert_error = { + func = 'crud.upsert', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'upsert', + expect_error = true, + }, + upsert_object_error = { + func = 'crud.upsert_object', + args = { space_name, { 'id' }, {{'+', 'age', 1}} }, + op = 'upsert', + expect_error = true, + }, + delete_error = { + func = 'crud.delete', + args = { space_name, { 'id' } }, + op = 'delete', + expect_error = true, + }, + min_error = { + func = 'crud.min', + args = { space_name, 'badindex' }, + op = 'borders', + expect_error = true, + }, + max_error = { + func = 'crud.max', + args = { space_name, 'badindex' }, + op = 'borders', + expect_error = true, + }, +} + +for name, case in pairs(simple_operation_cases) do + local test_name = ('test_%s'):format(name) + + if case.prepare ~= nil then + g.before_test(test_name, case.prepare) + end + + g[test_name] = function(g) + -- Collect stats before call. + local stats_before = g:get_stats(space_name) + t.assert_type(stats_before, 'table') + + -- Call operation. + local before_start = clock.monotonic() + + local _, err + if case.eval ~= nil then + if case.pcall then + _, err = pcall(g.router.eval, g.router, case.eval, case.args) + else + _, err = g.router:eval(case.eval, case.args) + end + else + _, err = g.router:call(case.func, case.args) + end + + local after_finish = clock.monotonic() + + if case.expect_error ~= true then + t.assert_equals(err, nil) + else + t.assert_not_equals(err, nil) + end + + -- Collect stats after call. + local stats_after = g:get_stats(space_name) + t.assert_type(stats_after, 'table') + t.assert_not_equals(stats_after[case.op], nil) + + -- Expecting 'ok' metrics to change on `expect_error == false` + -- or 'error' to change otherwise. + local changed, unchanged + if case.expect_error == true then + changed = 'error' + unchanged = 'ok' + else + unchanged = 'error' + changed = 'ok' + end + + local op_before = get_before_stats(stats_before, case.op) + local changed_before = op_before[changed] + local changed_after = stats_after[case.op][changed] + + t.assert_equals(changed_after.count - changed_before.count, 1, + 'Expected count incremented') + + local ok_latency_max = math.max(changed_before.latency, after_finish - before_start) + + t.assert_gt(changed_after.latency, 0, + 'Changed latency has appropriate value') + t.assert_le(changed_after.latency, ok_latency_max, + 'Changed latency has appropriate value') + + local time_diff = changed_after.time - changed_before.time + + t.assert_gt(time_diff, 0, 'Total time increase has appropriate value') + t.assert_le(time_diff, after_finish - before_start, + 'Total time increase has appropriate value') + + local unchanged_before = op_before[unchanged] + local unchanged_after = stats_after[case.op][unchanged] + + t.assert_equals(unchanged_before, unchanged_after, 'Other stats remained the same') + end +end + +-- Call some non-select operations on non-existing +-- spaces and ensure statistics is updated. +local unknown_space_cases = { + insert = { + func = 'crud.insert', + args = { unknown_space_name, {} }, + op = 'insert', + }, + insert_object = { + func = 'crud.insert_object', + args = { unknown_space_name, {} }, + op = 'insert', + }, + get = { + func = 'crud.get', + args = { unknown_space_name, {} }, + op = 'get', + }, + select = { + func = 'crud.select', + args = { unknown_space_name, {} }, + op = 'select', + }, + pairs = { + eval = eval.pairs_pcall, + args = { unknown_space_name, {} }, + op = 'select', + }, + replace = { + func = 'crud.replace', + args = { unknown_space_name, {} }, + op = 'replace', + }, + replace_object = { + func = 'crud.replace_object', + args = { unknown_space_name, {}, {} }, + op = 'replace', + }, + update = { + func = 'crud.update', + args = { unknown_space_name, {}, {} }, + op = 'update', + }, + upsert = { + func = 'crud.upsert', + args = { unknown_space_name, {}, {} }, + op = 'upsert', + }, + upsert_object = { + func = 'crud.upsert_object', + args = { unknown_space_name, {}, {} }, + op = 'upsert', + }, + delete = { + func = 'crud.delete', + args = { unknown_space_name, {} }, + op = 'delete', + }, + truncate = { + func = 'crud.truncate', + args = { unknown_space_name }, + op = 'truncate', + }, + len = { + func = 'crud.len', + args = { unknown_space_name }, + op = 'len', + }, + min = { + func = 'crud.min', + args = { unknown_space_name }, + op = 'borders', + }, + max = { + func = 'crud.max', + args = { unknown_space_name }, + op = 'borders', + }, +} + +for name, case in pairs(unknown_space_cases) do + local test_name = ('test_%s_on_unknown_space'):format(name) + + g[test_name] = function(g) + -- Collect statss before call. + local stats_before = g:get_stats() + t.assert_type(stats_before, 'table') + + -- Call operation. + local _, err + if case.eval ~= nil then + _, err = g.router:eval(case.eval, case.args) + else + _, err = g.router:call(case.func, case.args) + end + + t.assert_not_equals(err, nil) + + -- Collect stats after call. + local stats_after = g:get_stats() + t.assert_type(stats_after, 'table') + + t.assert_equals(stats_after.space_not_found - stats_before.space_not_found, 1, + "space_not_found statistic incremented") + t.assert_equals(stats_after.spaces, stats_before.spaces, + "Existing spaces stats haven't changed") + end +end diff --git a/test/unit/stats_test.lua b/test/unit/stats_test.lua new file mode 100644 index 00000000..97a8d124 --- /dev/null +++ b/test/unit/stats_test.lua @@ -0,0 +1,443 @@ +local clock = require('clock') +local fio = require('fio') +local fun = require('fun') +local t = require('luatest') + +local stats_module = require('crud.stats.module') +local utils = require('crud.common.utils') + +local g = t.group('stats_unit') +local helpers = require('test.helper') + +local space_name = 'customers' +local unknown_space_name = 'non_existing_space' + +g.before_all(function(g) + -- Enable test cluster for "is space exist?" checks. + g.cluster = helpers.Cluster:new({ + datadir = fio.tempdir(), + server_command = helpers.entrypoint('srv_simple_operations'), + use_vshard = true, + replicasets = helpers.get_test_replicasets(), + }) + g.cluster:start() + g.router = g.cluster:server('router').net_box + + helpers.prepare_simple_functions(g.router) + g.router:eval("stats_module = require('crud.stats.module')") + + t.assert_equals(helpers.is_space_exist(g.router, space_name), true) + t.assert_equals(helpers.is_space_exist(g.router, unknown_space_name), false) +end) + +g.after_all(function(g) + helpers.stop_cluster(g.cluster) +end) + +-- Reset statistics between tests, reenable if needed. +g.before_each(function(g) + g:enable_stats() +end) + +g.after_each(function(g) + g:disable_stats() +end) + +function g:get_stats(space_name) + return self.router:eval("return stats_module.get(...)", { space_name }) +end + +function g:enable_stats() + self.router:eval("stats_module.enable()") +end + +function g:disable_stats() + self.router:eval("stats_module.disable()") +end + +function g:reset_stats() + self.router:eval("return stats_module.reset()") +end + +g.test_get_format_after_enable = function(g) + local stats = g:get_stats() + + t.assert_type(stats, 'table') + t.assert_equals(stats.spaces, {}) + t.assert_equals(stats.space_not_found, 0) +end + +g.test_get_by_space_name_format_after_enable = function(g) + local stats = g:get_stats(space_name) + + t.assert_type(stats, 'table') + t.assert_equals(stats, {}) +end + +-- Test statistics values after wrapped functions call +-- for existing space. +local observe_cases = { + wrapper_observes_expected_values_on_ok = { + operations = stats_module.op, + func = 'return_true', + changed_coll = 'ok', + unchanged_coll = 'error', + }, + wrapper_observes_expected_values_on_error_return = { + operations = stats_module.op, + func = 'return_err', + changed_coll = 'error', + unchanged_coll = 'ok', + }, + wrapper_observes_expected_values_on_error_throw = { + operations = stats_module.op, + func = 'throws_error', + changed_coll = 'error', + unchanged_coll = 'ok', + pcall = true, + }, + pairs_wrapper_observes_expected_values_on_ok = { + operations = { stats_module.op.SELECT }, + func = 'pairs_ok', + changed_coll = 'ok', + unchanged_coll = 'error', + opts = { pairs = true }, + }, + pairs_wrapper_observes_expected_values_on_error = { + operations = { stats_module.op.SELECT }, + func = 'throws_error', + changed_coll = 'error', + unchanged_coll = 'ok', + pcall = true, + opts = { pairs = true }, + }, +} + +local call_wrapped = [[ + local func = rawget(_G, select(1, ...)) + local op = select(2, ...) + local opts = select(3, ...) + local space_name = select(4, ...) + + stats_module.wrap(func, op, opts)(space_name) +]] + +for name, case in pairs(observe_cases) do + for _, op in pairs(case.operations) do + local test_name = ('test_%s_%s'):format(op, name) + + g[test_name] = function(g) + -- Call wrapped functions on server side. + -- Collect execution times from outside. + local run_count = 10 + local time_diffs = {} + + local args = { case.func, op, case.opts, space_name } + + for _ = 1, run_count do + local before_start = clock.monotonic() + + if case.pcall then + pcall(g.router.eval, g.router, call_wrapped, args) + else + g.router:eval(call_wrapped, args) + end + + local after_finish = clock.monotonic() + + table.insert(time_diffs, after_finish - before_start) + end + + table.sort(time_diffs) + local total_time = fun.foldl(function(acc, x) return acc + x end, 0, time_diffs) + + -- Validate stats format after execution. + local total_stats = g:get_stats() + t.assert_type(total_stats, 'table', 'Total stats present after observations') + + local space_stats = g:get_stats(space_name) + t.assert_type(space_stats, 'table', 'Space stats present after observations') + + t.assert_equals(total_stats.spaces[space_name], space_stats, + 'Space stats is a section of total stats') + + local op_stats = space_stats[op] + t.assert_type(op_stats, 'table', 'Op stats present after observations for the space') + + -- Expected collectors (changed_coll: 'ok' or 'error') have changed. + local changed = op_stats[case.changed_coll] + t.assert_type(changed, 'table', 'Status stats present after observations') + + t.assert_equals(changed.count, run_count, 'Count incremented by count of runs') + + local sleep_time = helpers.simple_functions_params().sleep_time + t.assert_ge(changed.latency, sleep_time, 'Latency has appropriate value') + t.assert_le(changed.latency, time_diffs[#time_diffs], 'Latency has appropriate value') + + t.assert_ge(changed.time, sleep_time * run_count, + 'Total time increase has appropriate value') + t.assert_le(changed.time, total_time, 'Total time increase has appropriate value') + + -- Other collectors (unchanged_coll: 'error' or 'ok') + -- have been initialized and have default values. + local unchanged = op_stats[case.unchanged_coll] + t.assert_type(unchanged, 'table', 'Other status stats present after observations') + + t.assert_equals( + unchanged, + { + count = 0, + latency = 0, + time = 0 + }, + 'Other status collectors initialized after observations' + ) + end + end +end + +-- Test wrapper preserves return values. +local disable_stats_cases = { + stats_disable_before_wrap_ = { + before_wrap = 'stats_module.disable()', + after_wrap = '', + }, + stats_disable_after_wrap_ = { + before_wrap = '', + after_wrap = 'stats_module.disable()', + }, + [''] = { + before_wrap = '', + after_wrap = '', + }, +} + +local preserve_return_cases = { + wrapper_preserves_return_values_on_ok = { + func = 'return_true', + res = true, + err = nil, + }, + wrapper_preserves_return_values_on_error = { + func = 'return_err', + res = nil, + err = helpers.simple_functions_params().error, + }, +} + +local preserve_throw_cases = { + wrapper_preserves_error_throw = { + opts = { pairs = false }, + }, + pairs_wrapper_preserves_error_throw = { + opts = { pairs = true }, + }, +} + +for name_head, disable_case in pairs(disable_stats_cases) do + for name_tail, return_case in pairs(preserve_return_cases) do + local test_name = ('test_%s%s'):format(name_head, name_tail) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local eval = ([[ + local func = rawget(_G, select(1, ...)) + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op) + %s -- after_wrap + + return w_func(space_name) + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + local res, err = g.router:eval(eval, { return_case.func, op, space_name }) + + t.assert_equals(res, return_case.res, 'Wrapper preserves first return value') + t.assert_equals(err, return_case.err, 'Wrapper preserves second return value') + end + end + + local test_name = ('test_%spairs_wrapper_preserves_return_values'):format(name_head) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local input = { a = 'a', b = 'b' } + local eval = ([[ + local input = select(1, ...) + local func = function() return pairs(input) end + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op, { pairs = true }) + %s -- after_wrap + + local res = {} + for k, v in w_func(space_name) do + res[k] = v + end + + return res + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + local res = g.router:eval(eval, { input, op, space_name }) + + t.assert_equals(input, res, 'Wrapper preserves pairs return values') + end + + for name_tail, throw_case in pairs(preserve_throw_cases) do + local test_name = ('test_%s%s'):format(name_head, name_tail) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local eval = ([[ + local func = rawget(_G, 'throws_error') + local opts = select(1, ...) + local op = select(2, ...) + local space_name = select(3, ...) + + %s -- before_wrap + local w_func = stats_module.wrap(func, op, opts) + %s -- after_wrap + + w_func(space_name) + ]]):format(disable_case.before_wrap, disable_case.after_wrap) + + t.assert_error_msg_contains( + helpers.simple_functions_params().error_msg, + g.router.eval, g.router, eval, { throw_case.opts, op, space_name } + ) + end + end +end + +-- Test statistics values after wrapped functions call +-- for non-existing space. +local err_not_exist_msg = utils.space_doesnt_exist_msg(unknown_space_name) +local err_validation_msg = "Params validation failed" +local error_cases = { + -- If standartized utils.space_doesnt_exist_msg error + -- returned, space not found. + unknown_space_error_return = { + func = (" function(space_name) return nil, OpError:new(%q); end "):format(err_not_exist_msg), + msg = err_not_exist_msg, + }, + unknown_space_error_throw = { + func = (" function(space_name) OpError:assert(false, %q); end "):format(err_not_exist_msg), + msg = err_not_exist_msg, + throw = true, + }, + -- If error returned, space is not in stats registry and + -- is unknown to vshard, space not found. + arbitrary_error_return_for_unknown_space = { + func = (" function(space_name) return nil, OpError:new(%q); end "):format(err_validation_msg), + msg = err_validation_msg, + }, + arbitrary_error_throw_for_unknown_space = { + func = (" function(space_name) OpError:assert(false, %q); end "):format(err_validation_msg), + msg = err_validation_msg, + throw = true, + }, +} + +for name, case in pairs(error_cases) do + local test_name = ('test_%s_increases_space_not_found_count'):format(name) + + g[test_name] = function(g) + local op = stats_module.op.INSERT + + local eval = ([[ + local errors = require('errors') + local utils = require('crud.common.utils') + + local OpError = errors.new_class('OpError') + + local func = %s + local op = select(1, ...) + local space_name = select(2, ...) + + return stats_module.wrap(func, op)(space_name) + ]]):format(case.func) + + local err_msg + if case.throw then + local status, err = pcall(g.router.eval, g.router, eval, + { op, unknown_space_name }) + t.assert_equals(status, false) + err_msg = tostring(err) + else + local _, err = g.router:eval(eval, { op, unknown_space_name }) + err_msg = err.str + end + + t.assert_str_contains(err_msg, case.msg, "Error preserved") + + local stats = g:get_stats() + + t.assert_equals(stats.space_not_found, 1) + t.assert_equals(stats.spaces[unknown_space_name], nil, + "Non-existing space haven't generated stats section") + end +end + +g.test_stats_is_empty_after_disable = function(g) + g:disable_stats() + + local op = stats_module.op.INSERT + g.router:eval(call_wrapped, { 'return_true', op, {}, space_name }) + + local stats = g:get_stats() + t.assert_equals(stats, {}) +end + +local function prepare_non_default_stats(g) + local op = stats_module.op.INSERT + g.router:eval(call_wrapped, { 'return_true', op, {}, space_name }) + + local stats = g:get_stats(space_name) + t.assert_equals(stats[op].ok.count, 1, 'Non-zero stats prepared') + + return stats +end + +g.test_enable_is_idempotent = function(g) + local stats_before = prepare_non_default_stats(g) + + g:enable_stats() + + local stats_after = g:get_stats(space_name) + + t.assert_equals(stats_after, stats_before, 'Stats have not been reset') +end + +g.test_reset = function(g) + prepare_non_default_stats(g) + + g:reset_stats() + + local stats = g:get_stats(space_name) + + t.assert_equals(stats, {}, 'Stats have been reset') +end + +g.test_reset_for_disabled_stats_does_not_init_module = function(g) + g:disable_stats() + + local stats_before = g:get_stats() + t.assert_equals(stats_before, {}, "Stats is empty") + + g:reset_stats() + + local stats_after = g:get_stats() + t.assert_equals(stats_after, {}, "Stats is still empty") +end + +g.test_enabling_stats_on_non_router_throws_error = function(g) + local storage = g.cluster:server('s1-master').net_box + t.assert_error(storage.eval, storage, " require('crud.stats.module').enable() ") +end