From f3eadc178d93cd8adaf6abd1cfac798237e8548e Mon Sep 17 00:00:00 2001 From: Igor Zolotarev <63460867+yngvar-antonsson@users.noreply.github.com> Date: Sat, 27 Apr 2024 17:25:29 +0200 Subject: [PATCH] Add new metrics (#483) Co-authored-by: Georgy Moiseev --- .github/workflows/test.yml | 4 +++ CHANGELOG.md | 8 ++++++ doc/monitoring/metrics_reference.rst | 37 +++++++++++++++++++++++++++ metrics/tarantool/memtx.lua | 30 ++++++++++++++++++++++ metrics/tarantool/vinyl.lua | 7 +++++ test/tarantool/memtx_metrics_test.lua | 31 ++++++++++++++++++++++ test/tarantool/vinyl_test.lua | 4 ++- 7 files changed, 120 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7324622e..632b87af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,10 @@ jobs: cartridge: "2.9.0" - tarantool: "2.11" cartridge: "2.10.0" + - tarantool: "3.0" + cartridge: "" + - tarantool: "3.1" + cartridge: "" runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ac20d27..c024b6ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Inconsistent metrics descrtiptions for memtx metrics. +- New memory metrics: + - tnt_memtx_tuples_data_total + - tnt_memtx_tuples_data_read_view + - tnt_memtx_tuples_data_garbage + - tnt_memtx_index_total + - tnt_memtx_index_read_view + - tnt_vinyl_memory_tuple + ### Deprecated - Using `cdata` values with `histogram:observe`. diff --git a/doc/monitoring/metrics_reference.rst b/doc/monitoring/metrics_reference.rst index 1e5c58ed..4909d7ec 100644 --- a/doc/monitoring/metrics_reference.rst +++ b/doc/monitoring/metrics_reference.rst @@ -665,6 +665,9 @@ The following metrics show state memory areas used by vinyl for caches and write * - ``tnt_vinyl_memory_bloom_filter`` - Amount of memory in bytes used by :ref:`bloom filters `. + * - ``tnt_vinyl_memory_tuple`` + - Total size of memory in bytes occupied by Vinyl tuples. + It includes cached tuples and tuples pinned by the Lua world. .. _metrics-reference-vinyl_scheduler: @@ -956,3 +959,37 @@ So ``stories`` and ``retained`` need to be measured separately. Number of retained ``tracking`` tuples / number of stories. * ``total`` Amount of bytes used by retained ``tracking`` tuples. + + +~~~~~~~~~~~~~~~~~~~~ +Read view statistics +~~~~~~~~~~~~~~~~~~~~ + + +.. container:: table + + .. list-table:: + :widths: 25 75 + :header-rows: 0 + + * - ``tnt_memtx_tuples_data_total`` + - Total amount of memory (in bytes) allocated for data tuples. + This includes ``tnt_memtx_tuples_data_read_view`` and + ``tnt_memtx_tuples_data_garbage`` metric values plus tuples that + are actually stored in memtx spaces. + + * - ``tnt_memtx_tuples_data_read_view`` + - Memory (in bytes) held for read views. + + * - ``tnt_memtx_tuples_data_garbage`` + - Memory (in bytes) that is unused and scheduled to be freed + (freed lazily on memory allocation). + + * - ``tnt_memtx_index_total`` + - Total amount of memory (in bytes) allocated for indexing data. + This includes ``tnt_memtx_index_read_view`` metric value + plus memory used for indexing tuples + that are actually stored in memtx spaces. + + * - ``tnt_memtx_index_read_view`` + - Memory (in bytes) held for read views. diff --git a/metrics/tarantool/memtx.lua b/metrics/tarantool/memtx.lua index 1cdf1bdb..937fa3de 100644 --- a/metrics/tarantool/memtx.lua +++ b/metrics/tarantool/memtx.lua @@ -122,6 +122,36 @@ local function update() 'Number of retained `read_view` tuples / number of stories', memtx_stat.mvcc.tuples.read_view.retained.total, {kind = "total"}, nil, {default = true}) + -- Tarantool 3.0 memory statistics + + local ok, memtx_stat_3 = pcall(box.stat.memtx) + if not ok or memtx_stat_3.data == nil or memtx_stat_3.index == nil then + return + end + + collectors_list.memtx_tuples_data_total = + utils.set_gauge('memtx_tuples_data_total', + 'Total amount of memory allocated for data tuples', + memtx_stat_3.data.total, nil, nil, {default = true}) + collectors_list.memtx_tuples_data_read_view = + utils.set_gauge('memtx_tuples_data_read_view', + 'Memory held for read views', + memtx_stat_3.data.read_view, nil, nil, {default = true}) + collectors_list.memtx_tuples_data_garbage = + utils.set_gauge('memtx_tuples_data_garbage', + 'Memory that is unused and scheduled to be freed', + memtx_stat_3.data.garbage, nil, nil, {default = true}) + + + collectors_list.memtx_index_total = + utils.set_gauge('memtx_index_total', + 'Total amount of memory allocated for indexing data', + memtx_stat_3.index.total, nil, nil, {default = true}) + collectors_list.memtx_index_read_view = + utils.set_gauge('memtx_index_read_view', + 'Memory held for read views', + memtx_stat_3.index.read_view, nil, nil, {default = true}) + end return { diff --git a/metrics/tarantool/vinyl.lua b/metrics/tarantool/vinyl.lua index 7a9ea34d..9348352f 100644 --- a/metrics/tarantool/vinyl.lua +++ b/metrics/tarantool/vinyl.lua @@ -60,6 +60,13 @@ local function update() utils.set_gauge('vinyl_memory_bloom_filter', 'Size of bloom filter', vinyl_stat.memory.bloom_filter, nil, nil, {default = true}) + if vinyl_stat.memory.tuple ~= nil then + collectors_list.vinyl_memory_tuple = + utils.set_gauge('vinyl_memory_tuple', + 'Total size of memory in bytes occupied by Vinyl tuples', + vinyl_stat.memory.tuple, nil, nil, {default = true}) + end + collectors_list.vinyl_scheduler_tasks = utils.set_gauge('vinyl_scheduler_tasks', 'Vinyl tasks count', vinyl_stat.scheduler.tasks_inprogress, {status = 'inprogress'}, nil, {default = true}) diff --git a/test/tarantool/memtx_metrics_test.lua b/test/tarantool/memtx_metrics_test.lua index ce042127..91aa4840 100644 --- a/test/tarantool/memtx_metrics_test.lua +++ b/test/tarantool/memtx_metrics_test.lua @@ -63,3 +63,34 @@ g.test_memtx = function(cg) end end) end + +g.test_memtx_read_view = function(cg) + t.skip_if(utils.is_version_less(_TARANTOOL, '3.1.0'), + 'Tarantool version is must be v3.1.0 or greater') + + cg.server:exec(function() + local metrics = require('metrics') + local memtx = require('metrics.tarantool.memtx') + local utils = require('test.utils') -- luacheck: ignore 431 + + metrics.enable_default_metrics() + memtx.update() + local default_metrics = metrics.collect() + local log = require('log') + + local metrics_list = { + 'tnt_memtx_tuples_data_total', + 'tnt_memtx_tuples_data_read_view', + 'tnt_memtx_tuples_data_garbage', + 'tnt_memtx_index_total', + 'tnt_memtx_index_read_view', + } + + for _, item in ipairs(metrics_list) do + log.info('checking metric: ' .. item) + local metric = utils.find_metric(item, default_metrics) + t.assert(metric) + t.assert_type(metric[1].value, 'number') + end + end) +end diff --git a/test/tarantool/vinyl_test.lua b/test/tarantool/vinyl_test.lua index d67bbe8f..708e8f28 100644 --- a/test/tarantool/vinyl_test.lua +++ b/test/tarantool/vinyl_test.lua @@ -31,8 +31,10 @@ g.test_vinyl_metrics_present = function(cg) if utils.is_version_less(_TARANTOOL, '2.8.3') and utils.is_version_greater(_TARANTOOL, '2.0.0') then t.assert_equals(metrics_cnt, 19) - else + elseif utils.is_version_less(_TARANTOOL, '3.0.0') then t.assert_equals(metrics_cnt, 20) + else + t.assert_equals(metrics_cnt, 21) end end) end