prometheus: support disabling aggregation at query time

In Seastar, metrics can be defined with implicit aggregation by specific labels,
which occurs at query time. This feature is useful, for instance to define metrics
per shard or even more finely grained per an application-defined entity while reporting
them in a more aggregated manner, such as sum or histogram per server.

However, there are times when it is necessary to inspect the fine-grained metrics.
This can be achieved by adding `__aggregate__=false` to the query string. For example:
`http://localhost:9180/metrics?__aggregate__=false`

Closes #2219

doc/prometheus.md

@@ -54,6 +54,16 @@ To remove the help lines set `__help__=false`
 for example:
 `http://localhost:9180/metrics?__help__=false`
 
+## Aggregation
+In Seastar, metrics can be defined with implicit aggregation by specific labels,
+which occurs at query time. This feature is useful, for instance to define metrics
+per shard or even more finely grained per an application-defined entity while reporting
+them in a more aggregated manner, such as sum or histogram per server.
+
+However, there are times when it is necessary to inspect the fine-grained metrics.
+This can be achieved by adding `__aggregate__=false` to the query string. For example:
+`http://localhost:9180/metrics?__aggregate__=false`
+
 ### Configuring the Prometheus server for picking specific metrics
 The [Prometheus configuration](https://prometheus.io/docs/prometheus/1.8/configuration/configuration/) describes the general Prometheus configuration.
 

src/core/prometheus.cc

@@ -735,15 +735,15 @@ std::string get_value_as_string(std::stringstream& s, const mi::metric_value& va
     return value_str;
 }
 
-future<> write_text_representation(output_stream<char>& out, const config& ctx, const metric_family_range& m, bool show_help, std::function<bool(const mi::labels_type&)> filter) {
-    return seastar::async([&ctx, &out, &m, show_help, filter] () mutable {
+future<> write_text_representation(output_stream<char>& out, const config& ctx, const metric_family_range& m, bool show_help, bool disable_aggregation, std::function<bool(const mi::labels_type&)> filter) {
+    return seastar::async([&ctx, &out, &m, show_help, disable_aggregation, filter] () mutable {
         bool found = false;
         std::stringstream s;
         for (metric_family& metric_family : m) {
             auto name = ctx.prefix + "_" + metric_family.name();
             found = false;
             metric_aggregate_by_labels aggregated_values(metric_family.metadata().aggregate_labels);
-            bool should_aggregate = !metric_family.metadata().aggregate_labels.empty();
+            bool should_aggregate = !disable_aggregation && !metric_family.metadata().aggregate_labels.empty();
             metric_family.foreach_metric([&s, &out, &ctx, &found, &name, &metric_family, &aggregated_values, should_aggregate, show_help, &filter](const mi::metric_value& value, const mi::metric_info& value_info) mutable {
                 s.clear();
                 s.str("");
@@ -788,12 +788,12 @@ future<> write_text_representation(output_stream<char>& out, const config& ctx,
     });
 }
 
-future<> write_protobuf_representation(output_stream<char>& out, const config& ctx, metric_family_range& m, std::function<bool(const mi::labels_type&)> filter) {
-    return do_for_each(m, [&ctx, &out, filter=std::move(filter)](metric_family& metric_family) mutable {
+future<> write_protobuf_representation(output_stream<char>& out, const config& ctx, metric_family_range& m, bool disable_aggregation, std::function<bool(const mi::labels_type&)> filter) {
+    return do_for_each(m, [&ctx, &out, disable_aggregation, filter=std::move(filter)](metric_family& metric_family) mutable {
         std::string s;
         google::protobuf::io::StringOutputStream os(&s);
         metric_aggregate_by_labels aggregated_values(metric_family.metadata().aggregate_labels);
-        bool should_aggregate = !metric_family.metadata().aggregate_labels.empty();
+        bool should_aggregate = !disable_aggregation && !metric_family.metadata().aggregate_labels.empty();
         auto& name = metric_family.name();
         pm::MetricFamily mtf;
         bool empty_metric = true;
@@ -896,15 +896,16 @@ class metrics_handler : public httpd::handler_base  {
         sstring metric_family_name = req->get_query_param("__name__");
         bool prefix = trim_asterisk(metric_family_name);
         bool show_help = req->get_query_param("__help__") != "false";
+        bool disable_aggregation = req->get_query_param("__aggregate__") != "false";
         std::function<bool(const mi::labels_type&)> filter = make_filter(*req);
-        rep->write_body(is_protobuf_format ? "proto" : "txt", [this, is_protobuf_format, metric_family_name, prefix, show_help, filter] (output_stream<char>&& s) {
+        rep->write_body(is_protobuf_format ? "proto" : "txt", [this, is_protobuf_format, metric_family_name, prefix, show_help, disable_aggregation, filter] (output_stream<char>&& s) {
             return do_with(metrics_families_per_shard(), output_stream<char>(std::move(s)),
-                    [this, is_protobuf_format, prefix, &metric_family_name, show_help, filter] (metrics_families_per_shard& families, output_stream<char>& s) mutable {
-                return get_map_value(families).then([&s, &families, this, is_protobuf_format, prefix, &metric_family_name, show_help, filter]() mutable {
+                    [this, is_protobuf_format, prefix, &metric_family_name, show_help, disable_aggregation, filter] (metrics_families_per_shard& families, output_stream<char>& s) mutable {
+                return get_map_value(families).then([&s, &families, this, is_protobuf_format, prefix, &metric_family_name, show_help, disable_aggregation, filter]() mutable {
                     return do_with(get_range(families, metric_family_name, prefix),
-                            [&s, this, is_protobuf_format, show_help, filter](metric_family_range& m) {
-                        return (is_protobuf_format) ?  write_protobuf_representation(s, _ctx, m, filter) :
-                                write_text_representation(s, _ctx, m, show_help, filter);
+                            [&s, this, is_protobuf_format, show_help, disable_aggregation, filter](metric_family_range& m) {
+                        return (is_protobuf_format) ?  write_protobuf_representation(s, _ctx, m, disable_aggregation, filter) :
+                                write_text_representation(s, _ctx, m, show_help, disable_aggregation, filter);
                     });
                 }).finally([&s] () mutable {
                     return s.close();