From e1fc020d516a5f67958a0a7502b04e7b5e1476e0 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 15 Feb 2023 19:54:56 +0300 Subject: [PATCH] plugin: flight recorder export Add plugin to export up-to-date metrics with their aggregates to Flight recorder. Plugin API contains of two handles: export() return extended format metrics table, plain_format(output) allows to get human-readable minimal useful metrics info. Part of tarantool/tarantool#7725 Part of tarantool/tarantool#7728 --- CHANGELOG.md | 1 + doc/monitoring/plugins.rst | 48 ++++++++++++++++ metrics-scm-1.rockspec | 1 + metrics/plugins/flight_recorder.lua | 44 ++++++++++++++ metrics/stash.lua | 6 +- test/plugins/flight_recorder_test.lua | 83 +++++++++++++++++++++++++++ 6 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 metrics/plugins/flight_recorder.lua create mode 100644 test/plugins/flight_recorder_test.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 229697a0..8465a992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - per second rate for counters; - min and max for gauges; - average for histograms and summaries; +- plugin to export extended format data with aggregates to Flight recorder ### Changed - Setup cartridge hotreload inside the role diff --git a/doc/monitoring/plugins.rst b/doc/monitoring/plugins.rst index 6083a48c..6c465523 100644 --- a/doc/monitoring/plugins.rst +++ b/doc/monitoring/plugins.rst @@ -199,6 +199,54 @@ Use the JSON plugin with Tarantool ``http.server`` as follows: end ) +Flight recorder +~~~~~~~~~~~~~~~ + +Usage +^^^^^ + +Import the plugin: + +.. code-block:: lua + + local flight_recorder_exporter = require('metrics.plugins.flight_recorder') + +.. module:: metrics.plugins.flight_recorder + +.. function:: export() + + :return: extended format output with aggregates + + .. code-block:: yaml + + - tnt_net_per_thread_connections_mingauge: + name: tnt_net_per_thread_connections_min + name_prefix: tnt_net_per_thread_connections + kind: gauge + metainfo: + aggregate: true + default: true + timestamp: 1676478112824745 + observations: + '': + "thread\t1": + label_pairs: + thread: '1' + value: 0 + ... + +.. function:: plain_format(output) + + :return: human-readable form of output + + .. code-block:: text + + tnt_info_memory_lua_max{"alias":"router-4"} 10237204 + tnt_info_memory_lua_min{"alias":"router-4"} 1921790 + tnt_info_memory_lua{"alias":"router-4"} 2733335 + tnt_info_uptime{"alias":"router-4"} 1052 + ... + .. _metrics-plugins-custom: Creating custom plugins diff --git a/metrics-scm-1.rockspec b/metrics-scm-1.rockspec index 64682d35..0461fd39 100644 --- a/metrics-scm-1.rockspec +++ b/metrics-scm-1.rockspec @@ -36,6 +36,7 @@ build = { ['metrics.plugins.graphite'] = 'metrics/plugins/graphite.lua', ['metrics.plugins.prometheus'] = 'metrics/plugins/prometheus.lua', ['metrics.plugins.json'] = 'metrics/plugins/json.lua', + ['metrics.plugins.flight_recorder'] = 'metrics/plugins/flight_recorder.lua', ['metrics.tarantool'] = 'metrics/tarantool.lua', ['metrics.tarantool.fibers'] = 'metrics/tarantool/fibers.lua', ['metrics.tarantool.info'] = 'metrics/tarantool/info.lua', diff --git a/metrics/plugins/flight_recorder.lua b/metrics/plugins/flight_recorder.lua new file mode 100644 index 00000000..af271383 --- /dev/null +++ b/metrics/plugins/flight_recorder.lua @@ -0,0 +1,44 @@ +local json = require('json') + +local metrics = require('metrics') +local stash = require('metrics.stash') +local string_utils = require('metrics.string_utils') + +local data_stash = stash.get('flight_recorder') + +local function export() + local output = metrics.collect{invoke_callbacks = true, extended_format = true} + local output_with_aggregates = metrics.compute_aggregates( + data_stash.output_with_aggregates_prev, output) + data_stash.output_with_aggregates_prev = output_with_aggregates + return output_with_aggregates +end + +local function string_sort(a, b) + return a:upper() < b:upper() +end + +local function plain_format(output) + local result = {} + for _, coll_obs in pairs(output) do + for group_name, obs_group in pairs(coll_obs.observations) do + local metric_name = string_utils.build_name(coll_obs.name, group_name) + for _, obs in pairs(obs_group) do + table.sort(obs.label_pairs) + table.insert(result, string.format('%s%s %s', + metric_name, + json.encode(obs.label_pairs), + obs.value + )) + end + end + end + + table.sort(result, string_sort) + return table.concat(result, '\n') +end + +return { + export = export, + plain_format = plain_format, +} diff --git a/metrics/stash.lua b/metrics/stash.lua index 2f5560e0..3b24154c 100644 --- a/metrics/stash.lua +++ b/metrics/stash.lua @@ -7,9 +7,13 @@ local stash = {} -- @tfield string cfg -- Stash for metrics module configuration. -- +-- @tfield string flight_recorder +-- Stash for flight recorder plugin data. +-- stash.name = { cfg = '__metrics_cfg', - cfg_internal = '__metrics_cfg_internal' + cfg_internal = '__metrics_cfg_internal', + flight_recorder = '__flight_recorder', } --- Setup Tarantool Cartridge reload. diff --git a/test/plugins/flight_recorder_test.lua b/test/plugins/flight_recorder_test.lua new file mode 100644 index 00000000..1babefe3 --- /dev/null +++ b/test/plugins/flight_recorder_test.lua @@ -0,0 +1,83 @@ +#!/usr/bin/env tarantool + +local json = require('json') + +local t = require('luatest') +local g = t.group('flight_recorder_plugin') + +local flight_recorder_exporter = require('metrics.plugins.flight_recorder') +local metrics = require('metrics') +local metrics_stash = require('metrics.stash') +local utils = require('test.utils') + +g.before_all(utils.init) + +g.before_each(function() + metrics_stash.get('flight_recorder').output_with_aggrergates_prev = nil + metrics.clear() + metrics.cfg{include = 'all', exclude = {}, labels = {}} +end) + +g.test_exporter = function() + metrics.counter('counter_test_total'):inc(1) + metrics.gauge('gauge_test'):set(1) + metrics.histogram('histogram_test'):observe(1) + metrics.summary('summary_test'):observe(1) + + local output_non_plugin = metrics.collect{invoke_callbacks = true, extended_format = true} + + local output_exporter_1 = flight_recorder_exporter.export() + local output_exporter_2 = flight_recorder_exporter.export() + local output_exporter_3 = flight_recorder_exporter.export() + + for key, _ in pairs(output_non_plugin) do + t.assert_type(output_exporter_1[key], 'table', + 'Default metric observation presents in exporter output') + end + + t.assert_gt(utils.len(output_exporter_1), utils.len(output_non_plugin), + 'Exporter observations is extended with aggregates (min, max, average)') + + t.assert_type(output_exporter_1['gauge_test_mingauge'], 'table', + 'Exporter observations is extended with min aggregates') + t.assert_type(output_exporter_1['gauge_test_maxgauge'], 'table', + 'Exporter observations is extended with max aggregates') + t.assert_type(output_exporter_1['histogram_test_averagegauge'], 'table', + 'Exporter observations is extended with histogram average aggregates') + t.assert_type(output_exporter_1['summary_test_averagegauge'], 'table', + 'Exporter observations is extended with summary average aggregates') + + t.assert_gt(utils.len(output_exporter_2), utils.len(output_exporter_1), + 'Exporter observations is extended with aggregates (rate)') + + t.assert_type(output_exporter_2['counter_test_per_secondgauge'], 'table', + 'Exporter observations is extended with rate aggregates') + + t.assert_equals(utils.len(output_exporter_3), utils.len(output_exporter_2), + 'Exporter observations contains the same set of aggregates after second collect') +end + +g.test_plain_format = function() + metrics.cfg{labels = {alias = 'router-4'}} + + metrics.counter('counter_test_total'):inc(1, {label = 'value'}) + metrics.gauge('gauge_test'):set(2, {label = 'value'}) + metrics.histogram('histogram_test'):observe(3, {label = 'value'}) + metrics.summary('summary_test'):observe(4, {label = 'value'}) + + local output = flight_recorder_exporter.export() + + local plain_format_output = flight_recorder_exporter.plain_format(output) + t.assert_str_contains(plain_format_output, + 'counter_test_total' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 1') + t.assert_str_contains(plain_format_output, + 'gauge_test' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 2') + t.assert_str_contains(plain_format_output, + 'histogram_test_count' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 1') + t.assert_str_contains(plain_format_output, + 'histogram_test_sum' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 3') + t.assert_str_contains(plain_format_output, + 'summary_test_count' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 1') + t.assert_str_contains(plain_format_output, + 'summary_test_sum' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 4') +end