From 36dc9bb63fa24f9f5adf96de08752d1f48888593 Mon Sep 17 00:00:00 2001
From: Georgy Moiseev <georgy.moiseev@corp.mail.ru>
Date: Fri, 24 Dec 2021 18:13:24 +0300
Subject: [PATCH] Support metrics integration without quantiles

Quantiles computations increases performance overhead by near 10% when
used in statistics. One may want to use statistics with metrics without
quantiles. The patch allows one to do it.

Patch also bumps minimal required metrics rock version.
`metrics >= 0.9.0` is required to use summary quantiles with
age buckets. `metrics >= 0.5.0, < 0.9.0` is unsupported
due to quantile overflow bug [1]. `metrics == 0.9.0` has bug that do
not permits to create summary collector without quantiles [2].
In fact, user may use `metrics >= 0.5.0`, `metrics != 0.9.0`
if he wants to use metrics without quantiles, and `metrics >= 0.9.0`
if he wants to use metrics with quantiles. But this is confusing,
so let's use a single restriction for both cases.

1. https://github.com/tarantool/metrics/issues/235
2. https://github.com/tarantool/metrics/issues/262

Follows up #224
---
 .github/workflows/test_on_push.yaml |   2 +-
 README.md                           |  14 +--
 crud/stats/local_registry.lua       |  43 +++++----
 crud/stats/metrics_registry.lua     | 133 +++++++++++++++++++++-------
 crud/stats/module.lua               |  19 ++--
 crud/stats/stash.lua                |  26 +-----
 test/helper.lua                     |  10 +++
 test/integration/stats_test.lua     |  53 +++++++----
 test/performance/perf_test.lua      |  14 ++-
 test/unit/stats_test.lua            |  15 +++-
 10 files changed, 226 insertions(+), 103 deletions(-)

diff --git a/.github/workflows/test_on_push.yaml b/.github/workflows/test_on_push.yaml
index d351aa7ac..eaa59f1f3 100644
--- a/.github/workflows/test_on_push.yaml
+++ b/.github/workflows/test_on_push.yaml
@@ -26,7 +26,7 @@ jobs:
           - tarantool-version: "2.8"
             metrics-version: "0.1.8"
           - tarantool-version: "2.8"
-            metrics-version: "0.9.0"
+            metrics-version: "0.10.0"
           - tarantool-version: "2.8"
             coveralls: true
             metrics-version: "0.12.0"
diff --git a/README.md b/README.md
index eb5df8ce1..16ec65f55 100644
--- a/README.md
+++ b/README.md
@@ -606,18 +606,21 @@ crud.disable_stats()
 crud.reset_stats()
 ```
 
-If [`metrics`](https://github.com/tarantool/metrics) `0.9.0` or greater
+If [`metrics`](https://github.com/tarantool/metrics) `0.10.0` or greater
 found, metrics collectors will be used by default to store statistics
 instead of local collectors. You can manually choose driver if needed.
 ```lua
--- Use metrics collectors.
+-- Use metrics collectors. (Default if metrics found).
 crud.enable_stats({ driver = 'metrics' })
 
+-- Use metrics collectors with 0.99 quantile.
+crud.enable_stats({ driver = 'metrics', quantiles = true })
+
 -- Use simple local collectors.
 crud.enable_stats({ driver = 'local' })
 ```
-Performance overhead is 3-5% in case of `local` driver and
-10-20% in case of `metrics` driver.
+Performance overhead is 3-7% in case of `local` driver and
+5-10% in case of `metrics` driver, up to 20% for `metrics` with quantiles.
 
 Format is as follows.
 ```
@@ -668,7 +671,8 @@ Each operation section contains of different collectors
 for success calls and error (both error throw and `nil, err`)
 returns. `count` is total requests count since instance start
 or stats restart. `latency` is 0.99 quantile of request execution
-time if `metrics` driver used, otherwise `latency` is total average.
+time if `metrics` driver used and quantiles enabled,
+otherwise `latency` is total average.
 `time` is total time of requests execution.
 
 In [`metrics`](https://www.tarantool.io/en/doc/latest/book/monitoring/)
diff --git a/crud/stats/local_registry.lua b/crud/stats/local_registry.lua
index b9db402cf..a322385db 100644
--- a/crud/stats/local_registry.lua
+++ b/crud/stats/local_registry.lua
@@ -1,10 +1,13 @@
+local errors = require('errors')
+
 local dev_checks = require('crud.common.dev_checks')
 local op_module = require('crud.stats.operation')
 local registry_common = require('crud.stats.registry_common')
 local stash = require('crud.stats.stash')
 
 local registry = {}
-local internal_registry = stash.get('local_registry')
+local internal = stash.get('local_registry')
+local StatsLocalError = errors.new_class('StatsLocalError', {capture_stack = false})
 
 --- Initialize local metrics registry
 --
@@ -12,12 +15,22 @@ local internal_registry = stash.get('local_registry')
 --  by users, init is not guaranteed to be idempotent.
 --
 -- @function init
+-- @tparam table opts
+--
+--  @tfield boolean quantiles
+--   Quantiles is not supported for local, only `false` is valid.
 --
 -- @treturn boolean Returns true.
 --
-function registry.init()
-    internal_registry.spaces = {}
-    internal_registry.space_not_found = 0
+function registry.init(opts)
+    dev_checks({ quantiles = 'boolean' })
+
+    StatsLocalError:assert(opts.quantiles == false,
+        "Quantiles are not supported for 'local' statistics registry")
+
+    internal.registry = {}
+    internal.registry.spaces = {}
+    internal.registry.space_not_found = 0
 
     return true
 end
@@ -32,7 +45,7 @@ end
 -- @treturn boolean Returns true.
 --
 function registry.destroy()
-    internal_registry = stash.reset('local_registry')
+    internal.registry = nil
 
     return true
 end
@@ -58,10 +71,10 @@ function registry.get(space_name)
     dev_checks('?string')
 
     if space_name ~= nil then
-        return table.deepcopy(internal_registry.spaces[space_name]) or {}
+        return table.deepcopy(internal.registry.spaces[space_name]) or {}
     end
 
-    return table.deepcopy(internal_registry)
+    return table.deepcopy(internal.registry)
 end
 
 --- Check if space statistics are present in registry
@@ -76,7 +89,7 @@ end
 function registry.is_unknown_space(space_name)
     dev_checks('string')
 
-    return internal_registry.spaces[space_name] == nil
+    return internal.registry.spaces[space_name] == nil
 end
 
 --- Increase requests count and update latency info
@@ -101,8 +114,8 @@ end
 function registry.observe(latency, space_name, op, status)
     dev_checks('number', 'string', 'string', 'string')
 
-    registry_common.init_collectors_if_required(internal_registry.spaces, space_name, op)
-    local collectors = internal_registry.spaces[space_name][op][status]
+    registry_common.init_collectors_if_required(internal.registry.spaces, space_name, op)
+    local collectors = internal.registry.spaces[space_name][op][status]
 
     collectors.count = collectors.count + 1
     collectors.time = collectors.time + latency
@@ -118,7 +131,7 @@ end
 -- @treturn boolean Returns true.
 --
 function registry.observe_space_not_found()
-    internal_registry.space_not_found = internal_registry.space_not_found + 1
+    internal.registry.space_not_found = internal.registry.space_not_found + 1
 
     return true
 end
@@ -142,8 +155,8 @@ function registry.observe_fetch(tuples_fetched, tuples_lookup, space_name)
     dev_checks('number', 'number', 'string')
 
     local op = op_module.SELECT
-    registry_common.init_collectors_if_required(internal_registry.spaces, space_name, op)
-    local collectors = internal_registry.spaces[space_name][op].details
+    registry_common.init_collectors_if_required(internal.registry.spaces, space_name, op)
+    local collectors = internal.registry.spaces[space_name][op].details
 
     collectors.tuples_fetched = collectors.tuples_fetched + tuples_fetched
     collectors.tuples_lookup = collectors.tuples_lookup + tuples_lookup
@@ -167,8 +180,8 @@ function registry.observe_map_reduces(count, space_name)
     dev_checks('number', 'string')
 
     local op = op_module.SELECT
-    registry_common.init_collectors_if_required(internal_registry.spaces, space_name, op)
-    local collectors = internal_registry.spaces[space_name][op].details
+    registry_common.init_collectors_if_required(internal.registry.spaces, space_name, op)
+    local collectors = internal.registry.spaces[space_name][op].details
 
     collectors.map_reduces = collectors.map_reduces + count
 
diff --git a/crud/stats/metrics_registry.lua b/crud/stats/metrics_registry.lua
index 830f6503b..9e6f560ec 100644
--- a/crud/stats/metrics_registry.lua
+++ b/crud/stats/metrics_registry.lua
@@ -7,7 +7,7 @@ local stash = require('crud.stats.stash')
 
 local registry = {}
 -- Used to cache collectors.
-local internal_registry = stash.get('metrics_registry')
+local internal = stash.get('metrics_registry')
 
 local metric_name = {
     -- Summary collector for all operations.
@@ -32,33 +32,40 @@ local LATENCY_QUANTILE = 0.99
 
 -- Increasing tolerance threshold affects performance.
 local DEFAULT_QUANTILES = {
-    [LATENCY_QUANTILE] = 1e-3,
+    [LATENCY_QUANTILE] = 1e-2,
 }
 
-local DEFAULT_SUMMARY_PARAMS = {
+local DEFAULT_AGE_PARAMS = {
     age_buckets_count = 2,
     max_age_time = 60,
 }
 
 --- Check if application supports metrics rock for registry
 --
---  `metrics >= 0.9.0` is required to use summary with
+--  `metrics >= 0.10.0` is required.
+--  `metrics >= 0.9.0` is required to use summary quantiles with
 --  age buckets. `metrics >= 0.5.0, < 0.9.0` is unsupported
 --  due to quantile overflow bug
 --  (https://github.com/tarantool/metrics/issues/235).
+--  `metrics == 0.9.0` has bug that do not permits
+--  to create summary collector without quantiles
+--  (https://github.com/tarantool/metrics/issues/262).
+--  In fact, user may use `metrics >= 0.5.0`, `metrics != 0.9.0`
+--  if he wants to use metrics without quantiles, and `metrics >= 0.9.0`
+--  if he wants to use metrics with quantiles. But this is confusing,
+--  so let's use a single restriction for both cases.
 --
 -- @function is_supported
 --
--- @treturn boolean Returns true if `metrics >= 0.9.0` found, false otherwise.
+-- @treturn boolean Returns true if `metrics >= 0.10.0` found, false otherwise.
 --
 function registry.is_supported()
     if is_package == false then
         return false
     end
 
-    -- Only metrics >= 0.9.0 supported.
-    local is_summary, summary = pcall(require, 'metrics.collectors.summary')
-    if is_summary == false or summary.rotate_age_buckets == nil then
+    -- Only metrics >= 0.10.0 supported.
+    if metrics.unregister_callback == nil then
         return false
     end
 
@@ -73,28 +80,45 @@ end
 --
 -- @function init
 --
+-- @tparam table opts
+--
+--  @tfield boolean quantiles
+--   If true, computes latency as 0.99 quantile with aging.
+--
 -- @treturn boolean Returns true.
 --
-function registry.init()
-    internal_registry[metric_name.stats] = metrics.summary(
+function registry.init(opts)
+    dev_checks({ quantiles = 'boolean' })
+
+    internal.opts = table.deepcopy(opts)
+
+    local quantile_params = nil
+    local age_params = nil
+    if opts.quantiles == true then
+        quantile_params = DEFAULT_QUANTILES
+        age_params = DEFAULT_AGE_PARAMS
+    end
+
+    internal.registry = {}
+    internal.registry[metric_name.stats] = metrics.summary(
         metric_name.stats,
         'CRUD router calls statistics',
-        DEFAULT_QUANTILES,
-        DEFAULT_SUMMARY_PARAMS)
+        quantile_params,
+        age_params)
 
-    internal_registry[metric_name.space_not_found] = metrics.counter(
+    internal.registry[metric_name.space_not_found] = metrics.counter(
         metric_name.space_not_found,
         'Spaces not found during CRUD calls')
 
-    internal_registry[metric_name.details.tuples_fetched] = metrics.counter(
+    internal.registry[metric_name.details.tuples_fetched] = metrics.counter(
         metric_name.details.tuples_fetched,
         'Tuples fetched from CRUD storages during select/pairs')
 
-    internal_registry[metric_name.details.tuples_lookup] = metrics.counter(
+    internal.registry[metric_name.details.tuples_lookup] = metrics.counter(
         metric_name.details.tuples_lookup,
         'Tuples looked up on CRUD storages while collecting response during select/pairs')
 
-    internal_registry[metric_name.details.map_reduces] = metrics.counter(
+    internal.registry[metric_name.details.map_reduces] = metrics.counter(
         metric_name.details.map_reduces,
         'Map reduces planned during CRUD select/pairs')
 
@@ -112,14 +136,58 @@ end
 -- @treturn boolean Returns true.
 --
 function registry.destroy()
-    for _, c in pairs(internal_registry) do
+    for _, c in pairs(internal.registry) do
         metrics.registry:unregister(c)
     end
 
-    internal_registry = stash.reset('metrics_registry')
+    internal.registry = nil
+    internal.opts = nil
+
     return true
 end
 
+--- Compute `latency` field of an observation
+--  If it is a { time = ..., count = ... } observation,
+--  compute latency as overall average and store it
+--  inside observation object.
+--
+-- @tparam table obs
+--  Objects from registry_common
+--  stats.spaces[name][op][status].
+--  If something like `details` collector
+--  passed, do nothing.
+--
+-- @function compute_obs_latency
+--
+local function compute_obs_latency(obs)
+    if obs.count == nil or obs.time == nil then
+        return
+    end
+
+    if obs.count == 0 then
+        obs.latency = 0
+    else
+        obs.latency = obs.time / obs.count
+    end
+end
+
+--- Compute `latency` field of each observation
+--  If quantiles disabled, we need to compute
+--  latency as overall average from `time` and
+--  `count` values.
+--
+-- @function compute_obs_latency
+--
+local function compute_latencies(stats)
+    for _, space_stats in pairs(stats.spaces) do
+        for _, op_stats in pairs(space_stats) do
+            for _, obs in pairs(op_stats) do
+                compute_obs_latency(obs)
+            end
+        end
+    end
+end
+
 --- Get copy of global metrics registry
 --
 --  Registries are not meant to used explicitly
@@ -145,7 +213,7 @@ function registry.get(space_name)
     }
 
     -- Fill operation basic statistics values.
-    for _, obs in ipairs(internal_registry[metric_name.stats]:collect()) do
+    for _, obs in ipairs(internal.registry[metric_name.stats]:collect()) do
         local op = obs.label_pairs.operation
         local status = obs.label_pairs.status
         local name = obs.label_pairs.name
@@ -157,6 +225,7 @@ function registry.get(space_name)
         registry_common.init_collectors_if_required(stats.spaces, name, op)
         local space_stats = stats.spaces[name]
 
+        -- metric_name.stats presents only if quantiles enabled.
         if obs.metric_name == metric_name.stats then
             if obs.label_pairs.quantile == LATENCY_QUANTILE then
                 space_stats[op][status].latency = obs.value
@@ -170,9 +239,13 @@ function registry.get(space_name)
         :: stats_continue ::
     end
 
+    if not internal.opts.quantiles then
+        compute_latencies(stats)
+    end
+
     -- Fill select/pairs detail statistics values.
     for stat_name, metric_name in pairs(metric_name.details) do
-        for _, obs in ipairs(internal_registry[metric_name]:collect()) do
+        for _, obs in ipairs(internal.registry[metric_name]:collect()) do
             local name = obs.label_pairs.name
             local op = obs.label_pairs.operation
 
@@ -191,7 +264,7 @@ function registry.get(space_name)
         return stats.spaces[space_name] or {}
     end
 
-    local _, obs = next(internal_registry[metric_name.space_not_found]:collect())
+    local _, obs = next(internal.registry[metric_name.space_not_found]:collect())
     if obs ~= nil then
         stats.space_not_found = obs.value
     end
@@ -211,7 +284,7 @@ end
 function registry.is_unknown_space(space_name)
     dev_checks('string')
 
-    for _, obs in ipairs(internal_registry[metric_name.stats]:collect()) do
+    for _, obs in ipairs(internal.registry[metric_name.stats]:collect()) do
         local name = obs.label_pairs.name
 
         if name == space_name then
@@ -220,7 +293,7 @@ function registry.is_unknown_space(space_name)
     end
 
     for _, metric_name in pairs(metric_name.details) do
-        for _, obs in ipairs(internal_registry[metric_name]:collect()) do
+        for _, obs in ipairs(internal.registry[metric_name]:collect()) do
             local name = obs.label_pairs.name
 
             if name == space_name then
@@ -259,7 +332,7 @@ function registry.observe(latency, space_name, op, status)
     -- Use `status` label to be consistent with `tnt_vinyl_*` and HTTP metrics labels.
     local label_pairs = { operation = op, name = space_name, status = status }
 
-    internal_registry[metric_name.stats]:observe(latency, label_pairs)
+    internal.registry[metric_name.stats]:observe(latency, label_pairs)
 
     return true
 end
@@ -271,7 +344,7 @@ end
 -- @treturn boolean Returns true.
 --
 function registry.observe_space_not_found()
-    internal_registry[metric_name.space_not_found]:inc(1)
+    internal.registry[metric_name.space_not_found]:inc(1)
 
     return true
 end
@@ -296,8 +369,8 @@ function registry.observe_fetch(tuples_fetched, tuples_lookup, space_name)
 
     local label_pairs = { name = space_name, operation = op_module.SELECT }
 
-    internal_registry[metric_name.details.tuples_fetched]:inc(tuples_fetched, label_pairs)
-    internal_registry[metric_name.details.tuples_lookup]:inc(tuples_lookup, label_pairs)
+    internal.registry[metric_name.details.tuples_fetched]:inc(tuples_fetched, label_pairs)
+    internal.registry[metric_name.details.tuples_lookup]:inc(tuples_lookup, label_pairs)
 
     return true
 end
@@ -318,7 +391,7 @@ function registry.observe_map_reduces(count, space_name)
     dev_checks('number', 'string')
 
     local label_pairs = { name = space_name, operation = op_module.SELECT }
-    internal_registry[metric_name.details.map_reduces]:inc(count, label_pairs)
+    internal.registry[metric_name.details.map_reduces]:inc(count, label_pairs)
 
     return true
 end
@@ -334,14 +407,14 @@ local function workaround_role_reload()
     end
 
     -- Check if this registry was enabled before reload.
-    if next(internal_registry) == nil then
+    if internal.registry == nil then
         return
     end
 
     -- Check if base collector is in metrics package registry.
     -- If it's not, then registry has beed cleaned up on role reload.
     if metrics.registry:find('summary', metric_name.stats) == nil then
-        registry.init()
+        registry.init(internal.opts)
     end
 end
 
diff --git a/crud/stats/module.lua b/crud/stats/module.lua
index 95e15c897..52dd20ff9 100644
--- a/crud/stats/module.lua
+++ b/crud/stats/module.lua
@@ -54,7 +54,7 @@ end
 -- @treturn boolean Returns true.
 --
 function stats.enable(opts)
-    checks({ driver = '?string' })
+    checks({ driver = '?string', quantiles = '?boolean' })
 
     StatsError:assert(
         rawget(_G, 'crud') ~= nil,
@@ -75,16 +75,23 @@ function stats.enable(opts)
         'Unsupported driver: %s', opts.driver
     )
 
-    if internal.driver == opts.driver then
+    if opts.quantiles == nil then
+        opts.quantiles = false
+    end
+
+    -- Do not reinit if called with same options.
+    if internal.opts ~= nil
+    and internal.opts.driver == opts.driver
+    and internal.opts.quantiles == opts.quantiles then
         return true
     end
 
     -- Disable old driver registry, if another one was requested.
     stats.disable()
 
-    internal.driver = opts.driver
+    internal.opts = table.deepcopy(opts)
     internal.registry = drivers[opts.driver]
-    internal.registry.init()
+    internal.registry.init({ quantiles = opts.quantiles })
 
     return true
 end
@@ -104,7 +111,7 @@ function stats.reset()
     end
 
     internal.registry.destroy()
-    internal.registry.init()
+    internal.registry.init({ quantiles = internal.opts.quantiles })
 
     return true
 end
@@ -124,7 +131,7 @@ function stats.disable()
 
     internal.registry.destroy()
     internal.registry = nil
-    internal.driver = nil
+    internal.opts = nil
 
     return true
 end
diff --git a/crud/stats/stash.lua b/crud/stats/stash.lua
index 2ae754152..bd3c182ca 100644
--- a/crud/stats/stash.lua
+++ b/crud/stats/stash.lua
@@ -2,10 +2,6 @@ local dev_checks = require('crud.common.dev_checks')
 
 local stash = {}
 
-local function get_G_name(name)
-    return ('__crud_stats_%s'):format(name)
-end
-
 --- Get a stash instance, initialize if needed
 --  Stashes are persistent to package reload and cartridge roles reload.
 --
@@ -19,7 +15,7 @@ end
 function stash.get(name)
     dev_checks('string')
 
-    local _G_name = get_G_name(name)
+    local _G_name = ('__crud_stats_%s'):format(name)
     local instance = rawget(_G, _G_name) or {}
     rawset(_G, _G_name, instance)
 
@@ -31,24 +27,4 @@ function stash.get(name)
     return instance
 end
 
---- Reset stash instance
---  Stashes are persistent to package reload and cartridge roles reload.
---
--- @function get
---
--- @tparam string name
---  Stash identifier.
---
--- @treturn table A stash instance.
---
-function stash.reset(name)
-    dev_checks('string')
-
-    local _G_name = get_G_name(name)
-    local instance = {}
-    rawset(_G, _G_name, instance)
-
-    return instance
-end
-
 return stash
diff --git a/test/helper.lua b/test/helper.lua
index 34fb679b9..6d07204b8 100644
--- a/test/helper.lua
+++ b/test/helper.lua
@@ -419,4 +419,14 @@ function helpers.reload_roles(srv)
     t.assert_equals({ok, err}, {true, nil})
 end
 
+function helpers.toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+
+    return nil
+end
+
 return helpers
diff --git a/test/integration/stats_test.lua b/test/integration/stats_test.lua
index e7ffb1402..6c206fcd9 100644
--- a/test/integration/stats_test.lua
+++ b/test/integration/stats_test.lua
@@ -6,10 +6,12 @@ local stats_registry_common = require('crud.stats.registry_common')
 
 local pgroup = t.group('stats_integration', {
     { driver = 'local' },
-    { driver = 'metrics' },
+    { driver = 'metrics', quantiles = false },
+    { driver = 'metrics', quantiles = true },
 })
 local group_metrics = t.group('stats_metrics_integration', {
-    { driver = 'metrics' },
+    { driver = 'metrics', quantiles = false },
+    { driver = 'metrics', quantiles = true },
 })
 local helpers = require('test.helper')
 
@@ -700,9 +702,12 @@ local function get_unique_label_values(metrics_data, label_key)
     return label_values
 end
 
-local function validate_stats(metrics)
-    local stats = find_metric('tnt_crud_stats', metrics)
-    t.assert_type(stats, 'table', '`tnt_crud_stats` summary metrics found')
+local function validate_stats(g, metrics)
+    local quantile_stats
+    if g.params.quantiles == true then
+        quantile_stats = find_metric('tnt_crud_stats', metrics)
+        t.assert_type(quantile_stats, 'table', '`tnt_crud_stats` summary metrics found')
+    end
 
     local stats_count = find_metric('tnt_crud_stats_count', metrics)
     t.assert_type(stats_count, 'table', '`tnt_crud_stats` summary metrics found')
@@ -714,8 +719,10 @@ local function validate_stats(metrics)
     local expected_operations = { 'insert', 'get', 'replace', 'update',
         'upsert', 'delete', 'select', 'truncate', 'len', 'borders' }
 
-    t.assert_items_equals(get_unique_label_values(stats, 'operation'), expected_operations,
-        'Metrics are labelled with operation')
+    if g.params.quantiles == true then
+        t.assert_items_equals(get_unique_label_values(quantile_stats, 'operation'), expected_operations,
+            'Metrics are labelled with operation')
+    end
 
     t.assert_items_equals(get_unique_label_values(stats_count, 'operation'), expected_operations,
         'Metrics are labelled with operation')
@@ -726,8 +733,12 @@ local function validate_stats(metrics)
 
     local expected_statuses = { 'ok', 'error' }
 
-    t.assert_items_equals( get_unique_label_values(stats, 'status'), expected_statuses,
-        'Metrics are labelled with status')
+    if g.params.quantiles == true then
+        t.assert_items_equals(
+            get_unique_label_values(quantile_stats, 'status'),
+            expected_statuses,
+            'Metrics are labelled with status')
+    end
 
     t.assert_items_equals(get_unique_label_values(stats_count, 'status'), expected_statuses,
         'Metrics are labelled with status')
@@ -738,8 +749,12 @@ local function validate_stats(metrics)
 
     local expected_names = { space_name }
 
-    t.assert_items_equals(get_unique_label_values(stats, 'name'), expected_names,
-        'Metrics are labelled with space name (only existing spaces)')
+    if g.params.quantiles == true then
+        t.assert_items_equals(
+            get_unique_label_values(quantile_stats, 'name'),
+            expected_names,
+            'Metrics are labelled with space name (only existing spaces)')
+    end
 
     t.assert_items_equals(get_unique_label_values(stats_count, 'name'),
         expected_names,
@@ -750,6 +765,12 @@ local function validate_stats(metrics)
         expected_names,
         'Metrics are labelled with space name (only existing spaces)')
 
+    if g.params.quantiles == true then
+        local expected_quantiles = { 0.99 }
+        t.assert_items_equals(get_unique_label_values(quantile_stats, 'quantile'), expected_quantiles,
+            'Quantile metrics presents')
+    end
+
 
     local tuples_fetched = find_metric('tnt_crud_tuples_fetched', metrics)
     t.assert_type(tuples_fetched, 'table', '`tnt_crud_tuples_fetched` metrics found')
@@ -826,7 +847,7 @@ group_metrics.before_test(
 
 group_metrics.test_stats_stored_in_global_metrics_registry = function(g)
     local metrics = get_metrics(g)
-    validate_stats(metrics)
+    validate_stats(g, metrics)
 end
 
 
@@ -895,11 +916,11 @@ group_metrics.before_test(
 group_metrics.test_stats_stored_in_metrics_registry_after_switch_to_metrics_driver = function(g)
     enable_stats(g, { driver = 'local' })
     -- Switch to metrics driver.
-    enable_stats(g, { driver = 'metrics' })
+    enable_stats(g)
 
     generate_stats(g)
     local metrics = get_metrics(g)
-    validate_stats(metrics)
+    validate_stats(g, metrics)
 end
 
 
@@ -947,7 +968,7 @@ group_metrics.test_role_reload_do_not_reset_metrics_observations = function(g)
     helpers.reload_roles(g.cluster:server('router'))
     g.router:eval("crud = require('crud')")
     local metrics = get_metrics(g)
-    validate_stats(metrics)
+    validate_stats(g, metrics)
 end
 
 
@@ -963,7 +984,7 @@ group_metrics.test_module_reload_do_not_reset_metrics_observations = function(g)
     ]])
 
     local metrics = get_metrics(g)
-    validate_stats(metrics)
+    validate_stats(g, metrics)
 end
 
 group_metrics.before_test(
diff --git a/test/performance/perf_test.lua b/test/performance/perf_test.lua
index 1e34af4f3..ad277496d 100644
--- a/test/performance/perf_test.lua
+++ b/test/performance/perf_test.lua
@@ -102,6 +102,7 @@ local column_name = {
     stats_disabled = 'stats disabled',
     local_stats = 'local stats',
     metrics_stats = 'metrics stats',
+    metrics_quantiles_stats = 'metrics stats (with quantiles)',
 }
 
 local function visualize_section(total_report, name, comment, section, params)
@@ -154,6 +155,7 @@ local function visualize_report(report)
         column_name.stats_disabled,
         column_name.local_stats,
         column_name.metrics_stats,
+        column_name.metrics_quantiles_stats,
     }
 
     local report_str = '\n==== PERFORMANCE REPORT ====\n\n\n'
@@ -225,10 +227,20 @@ local stats_cases = {
                 return require('crud.stats.metrics_registry').is_supported()
             ]])
             t.skip_if(is_metrics_supported == false, 'Metrics registry is unsupported')
-            g.router:call("crud.enable_stats", {{ driver = 'metrics' }})
+            g.router:call("crud.enable_stats", {{ driver = 'metrics', quantiles = false }})
         end,
         column_name = column_name.metrics_stats,
     },
+    metrics_quantiles_stats = {
+        prepare = function(g)
+            local is_metrics_supported = g.router:eval([[
+                return require('crud.stats.metrics_registry').is_supported()
+            ]])
+            t.skip_if(is_metrics_supported == false, 'Metrics registry is unsupported')
+            g.router:call("crud.enable_stats", {{ driver = 'metrics', quantiles = true }})
+        end,
+        column_name = column_name.metrics_quantiles_stats,
+    },
 }
 
 local integration_params = {
diff --git a/test/unit/stats_test.lua b/test/unit/stats_test.lua
index 48848c022..278c2ae06 100644
--- a/test/unit/stats_test.lua
+++ b/test/unit/stats_test.lua
@@ -8,7 +8,8 @@ local utils = require('crud.common.utils')
 
 local pgroup = t.group('stats_unit', {
     { driver = 'local' },
-    { driver = 'metrics' },
+    { driver = 'metrics', quantiles = false },
+    { driver = 'metrics', quantiles = true },
 })
 local group_driver = t.group('stats_driver_unit')
 local helpers = require('test.helper')
@@ -543,13 +544,14 @@ end
 group_driver.test_default_driver = function(g)
     enable_stats(g)
 
-    local driver = g.router:eval(" return stats_module.internal.driver ")
+    local opts = g.router:eval(" return stats_module.internal.opts ")
 
     if g.is_metrics_supported then
-        t.assert_equals(driver, 'metrics')
+        t.assert_equals(opts.driver, 'metrics')
     else
-        t.assert_equals(driver, 'local')
+        t.assert_equals(opts.driver, 'local')
     end
+    t.assert_equals(opts.quantiles, false)
 end
 
 group_driver.before_test(
@@ -588,3 +590,8 @@ group_driver.test_stats_enable_with_metrics_throws_error_if_unsupported = functi
         enable_stats, g, { driver = 'metrics' })
 end
 
+group_driver.test_stats_enable_with_local_throws_error_if_quantiles_enabled = function(g)
+    t.assert_error_msg_contains(
+        'Quantiles are not supported',
+        enable_stats, g, { driver = 'local', quantiles = true })
+end