plugin: flight recorder export

Add plugin to export up-to-date metrics with their aggregates to Flight
recorder. Plugin API contains of two handles: export() return extended
format metrics table, plain_format(output) allows to get human-readable
minimal useful metrics info.

Part of tarantool/tarantool#7725
Part of tarantool/tarantool#7728

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - per second rate for counters;
   - min and max for gauges;
   - average for histograms and summaries;
+- plugin to export extended format data with aggregates to Flight recorder
 
 ### Changed
 - Setup cartridge hotreload inside the role

doc/monitoring/plugins.rst

@@ -199,6 +199,54 @@ Use the JSON plugin with Tarantool ``http.server`` as follows:
         end
     )
 
+Flight recorder
+~~~~~~~~~~~~~~~
+
+Usage
+^^^^^
+
+Import the plugin:
+
+..  code-block:: lua
+
+    local flight_recorder_exporter = require('metrics.plugins.flight_recorder')
+
+..  module:: metrics.plugins.flight_recorder
+
+..  function:: export()
+
+    :return: extended format output with aggregates
+
+        ..  code-block:: yaml
+
+            - tnt_net_per_thread_connections_mingauge:
+                name: tnt_net_per_thread_connections_min
+                name_prefix: tnt_net_per_thread_connections
+                kind: gauge
+                metainfo:
+                  aggregate: true
+                  default: true
+                timestamp: 1676478112824745
+                observations:
+                  '':
+                    "thread\t1":
+                      label_pairs:
+                        thread: '1'
+                      value: 0
+            ...
+
+..  function:: plain_format(output)
+
+    :return: human-readable form of output
+
+        ..  code-block:: text
+
+            tnt_info_memory_lua_max{"alias":"router-4"} 10237204
+            tnt_info_memory_lua_min{"alias":"router-4"} 1921790
+            tnt_info_memory_lua{"alias":"router-4"} 2733335
+            tnt_info_uptime{"alias":"router-4"} 1052
+            ...
+
 .. _metrics-plugins-custom:
 
 Creating custom plugins

metrics-scm-1.rockspec

@@ -36,6 +36,7 @@ build = {
         ['metrics.plugins.graphite']        = 'metrics/plugins/graphite.lua',
         ['metrics.plugins.prometheus']      = 'metrics/plugins/prometheus.lua',
         ['metrics.plugins.json']            = 'metrics/plugins/json.lua',
+        ['metrics.plugins.flight_recorder'] = 'metrics/plugins/flight_recorder.lua',
         ['metrics.tarantool']               = 'metrics/tarantool.lua',
         ['metrics.tarantool.fibers']        = 'metrics/tarantool/fibers.lua',
         ['metrics.tarantool.info']          = 'metrics/tarantool/info.lua',

metrics/plugins/flight_recorder.lua

@@ -0,0 +1,44 @@
+local json = require('json')
+
+local metrics = require('metrics')
+local stash = require('metrics.stash')
+local string_utils = require('metrics.string_utils')
+
+local data_stash = stash.get('flight_recorder')
+
+local function export()
+    local output = metrics.collect{invoke_callbacks = true, extended_format = true}
+    local output_with_aggregates = metrics.compute_aggregates(
+        data_stash.output_with_aggregates_prev, output)
+    data_stash.output_with_aggregates_prev = output_with_aggregates
+    return output_with_aggregates
+end
+
+local function string_sort(a, b)
+    return a:upper() < b:upper()
+end
+
+local function plain_format(output)
+    local result = {}
+    for _, coll_obs in pairs(output) do
+        for group_name, obs_group in pairs(coll_obs.observations) do
+            local metric_name = string_utils.build_name(coll_obs.name, group_name)
+            for _, obs in pairs(obs_group) do
+                table.sort(obs.label_pairs)
+                table.insert(result, string.format('%s%s %s',
+                    metric_name,
+                    json.encode(obs.label_pairs),
+                    obs.value
+                ))
+            end
+        end
+    end
+
+    table.sort(result, string_sort)
+    return table.concat(result, '\n')
+end
+
+return {
+    export = export,
+    plain_format = plain_format,
+}

metrics/stash.lua

@@ -7,9 +7,13 @@ local stash = {}
 -- @tfield string cfg
 --  Stash for metrics module configuration.
 --
+-- @tfield string flight_recorder
+--  Stash for flight recorder plugin data.
+--
 stash.name = {
     cfg = '__metrics_cfg',
-    cfg_internal = '__metrics_cfg_internal'
+    cfg_internal = '__metrics_cfg_internal',
+    flight_recorder = '__flight_recorder',
 }
 
 --- Setup Tarantool Cartridge reload.

test/plugins/flight_recorder_test.lua

@@ -0,0 +1,83 @@
+#!/usr/bin/env tarantool
+
+local json = require('json')
+
+local t = require('luatest')
+local g = t.group('flight_recorder_plugin')
+
+local flight_recorder_exporter = require('metrics.plugins.flight_recorder')
+local metrics = require('metrics')
+local metrics_stash = require('metrics.stash')
+local utils = require('test.utils')
+
+g.before_all(utils.init)
+
+g.before_each(function()
+    metrics_stash.get('flight_recorder').output_with_aggrergates_prev = nil
+    metrics.clear()
+    metrics.cfg{include = 'all', exclude = {}, labels = {}}
+end)
+
+g.test_exporter = function()
+    metrics.counter('counter_test_total'):inc(1)
+    metrics.gauge('gauge_test'):set(1)
+    metrics.histogram('histogram_test'):observe(1)
+    metrics.summary('summary_test'):observe(1)
+
+    local output_non_plugin = metrics.collect{invoke_callbacks = true, extended_format = true}
+
+    local output_exporter_1 = flight_recorder_exporter.export()
+    local output_exporter_2 = flight_recorder_exporter.export()
+    local output_exporter_3 = flight_recorder_exporter.export()
+
+    for key, _ in pairs(output_non_plugin) do
+        t.assert_type(output_exporter_1[key], 'table',
+            'Default metric observation presents in exporter output')
+    end
+
+    t.assert_gt(utils.len(output_exporter_1), utils.len(output_non_plugin),
+        'Exporter observations is extended with aggregates (min, max, average)')
+
+    t.assert_type(output_exporter_1['gauge_test_mingauge'], 'table',
+        'Exporter observations is extended with min aggregates')
+    t.assert_type(output_exporter_1['gauge_test_maxgauge'], 'table',
+        'Exporter observations is extended with max aggregates')
+    t.assert_type(output_exporter_1['histogram_test_averagegauge'], 'table',
+        'Exporter observations is extended with histogram average aggregates')
+    t.assert_type(output_exporter_1['summary_test_averagegauge'], 'table',
+        'Exporter observations is extended with summary average aggregates')
+
+    t.assert_gt(utils.len(output_exporter_2), utils.len(output_exporter_1),
+        'Exporter observations is extended with aggregates (rate)')
+
+    t.assert_type(output_exporter_2['counter_test_per_secondgauge'], 'table',
+        'Exporter observations is extended with rate aggregates')
+
+    t.assert_equals(utils.len(output_exporter_3), utils.len(output_exporter_2),
+        'Exporter observations contains the same set of aggregates after second collect')
+end
+
+g.test_plain_format = function()
+    metrics.cfg{labels = {alias = 'router-4'}}
+
+    metrics.counter('counter_test_total'):inc(1, {label = 'value'})
+    metrics.gauge('gauge_test'):set(2, {label = 'value'})
+    metrics.histogram('histogram_test'):observe(3, {label = 'value'})
+    metrics.summary('summary_test'):observe(4, {label = 'value'})
+
+    local output = flight_recorder_exporter.export()
+
+    local plain_format_output = flight_recorder_exporter.plain_format(output)
+    t.assert_str_contains(plain_format_output,
+        'counter_test_total' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 1')
+    t.assert_str_contains(plain_format_output,
+        'gauge_test' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 2')
+    t.assert_str_contains(plain_format_output,
+        'histogram_test_count' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 1')
+    t.assert_str_contains(plain_format_output,
+        'histogram_test_sum' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 3')
+    t.assert_str_contains(plain_format_output,
+        'summary_test_count' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 1')
+    t.assert_str_contains(plain_format_output,
+        'summary_test_sum' .. json.encode({alias = 'router-4', label = 'value'}) .. ' 4')
+end