Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: introduce data sink to collect snapshot from each metric periodically #1117

Open
wants to merge 24 commits into
base: master
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
98efadf
feat: take snapshots from metrics and export to data sink
empiredan Aug 1, 2022
73c1796
feat: take snapshots from metrics and export to data sink
empiredan Aug 1, 2022
0a9e3ac
feat: take snapshots from metrics and export to data sink
empiredan Aug 1, 2022
4d1e33b
feat: take snapshots from metrics and export to data sink
empiredan Aug 3, 2022
a8ac596
feat: take snapshots from metrics and export to data sink
empiredan Aug 3, 2022
e1c6bfd
feat: take snapshots from metrics and export to data sink
empiredan Aug 3, 2022
017f945
feat: take snapshots from metrics and export to data sink
empiredan Aug 3, 2022
37fc291
feat: take snapshots from metrics and export to data sink
empiredan Aug 3, 2022
c9629f0
feat: take snapshots from metrics and export to data sink
empiredan Aug 4, 2022
e04d72e
feat: take snapshots from metrics and export to data sink
empiredan Aug 4, 2022
1b9f4af
feat: take snapshots from metrics and export to data sink
empiredan Aug 4, 2022
4768e58
feat: take snapshots from metrics and export to data sink
empiredan Aug 4, 2022
1220915
feat: take snapshots from metrics and export to data sink
empiredan Aug 5, 2022
83200e1
feat: take snapshots from metrics and export to data sink
empiredan Aug 5, 2022
85f3f26
feat: take snapshots from metrics and export to data sink
empiredan Aug 9, 2022
c92ef2d
feat: take snapshots from metrics and export to data sink
empiredan Aug 10, 2022
3c1f503
feat: take snapshots from metrics and export to data sink
empiredan Aug 11, 2022
5be088c
feat: take snapshots from metrics and export to data sink
empiredan Aug 11, 2022
3b3d9dc
feat: take snapshots from metrics and export to data sink
empiredan Aug 11, 2022
f108b67
feat: take snapshots from metrics and export to data sink
empiredan Aug 11, 2022
7e6393a
feat: introduce data sink to collect snapshot from each metric period…
empiredan Aug 12, 2022
5881d9b
feat: introduce data sink to collect snapshot from each metric period…
empiredan Aug 12, 2022
36fac25
feat: introduce data sink to collect snapshot from each metric period…
empiredan Aug 12, 2022
eb116eb
feat: introduce data sink to collect snapshot from each metric period…
empiredan Aug 15, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: take snapshots from metrics and export to data sink
empiredan committed Aug 10, 2022
commit c92ef2dea06aeb806b0a179bdaa6f39806ad6861
2 changes: 2 additions & 0 deletions src/rdsn/include/dsn/utility/metrics.h
Original file line number Diff line number Diff line change
@@ -769,6 +769,8 @@ inline size_t kth_percentile_to_nth_index(size_t size, kth_percentile_type type)
std::set<kth_percentile_type> get_all_kth_percentile_types();
const std::set<kth_percentile_type> kAllKthPercentileTypes = get_all_kth_percentile_types();

std::set<std::string> kth_percentiles_to_labels(const std::set<kth_percentile_type> &kth_percentiles = kAllKthPercentileTypes);

// The percentile is a metric type that samples observations. The size of samples has an upper
// bound. Once the maximum size is reached, the earliest observations will be overwritten.
//
10 changes: 10 additions & 0 deletions src/rdsn/src/utils/metrics.cpp
Original file line number Diff line number Diff line change
@@ -44,6 +44,16 @@ std::set<kth_percentile_type> get_all_kth_percentile_types()
return all_types;
}

std::set<std::string> kth_percentiles_to_labels(const std::set<kth_percentile_type> &kth_percentiles)
{
std::set<std::string> labels;
std::transform(kth_percentiles.begin(), kth_percentiles.end(), std::inserter(labels, labels.begin()),
[](const kth_percentile_type &type) {
return kKthLabels[static_cast<size_t>(type)];
});
return labels;
}

metric_entity::metric_entity(const std::string &id, attr_map &&attrs)
: _id(id), _lock(), _attrs(std::move(attrs)), _metrics()
{
37 changes: 35 additions & 2 deletions src/rdsn/src/utils/test/metrics_test.cpp
Original file line number Diff line number Diff line change
@@ -1009,6 +1009,29 @@ TEST_F(metrics_test, percentile_double)
floating_checker<value_type>>(METRIC_test_percentile_double);
}

void build_expected_snapshots(metric *m, metric_snapshot::value_type value, const metric_entity::attr_map &attrs, my_data_sink::metric_map &expected_metrics)
{
metric_snapshot snapshot(m->prototype()->name(), m->prototype()->type(), value, metric_snapshot::attr_map(attrs));
expected_metrics[m->prototype()->name()][snapshot.encode_attributes()] = std::move(snapshot);
}

void build_expected_snapshots(metric *m, metric_snapshot::value_type value, const metric_entity::attr_map &attrs, metric_snapshot::value_type increase, my_data_sink::counter_map &expected_counters)
{
counter_snapshot snapshot(m->prototype()->name(), m->prototype()->type(), value, metric_snapshot::attr_map(attrs), increase);
expected_counters[m->prototype()->name()][snapshot.encode_attributes()] = std::move(snapshot);
}

void build_expected_snapshots(metric *m, metric_snapshot::value_type value, const metric_entity::attr_map &attrs, const std::set<kth_percentile_type> &kth_percentiles, my_data_sink::metric_map &expected_metrics)
{
auto kth_labels = kth_percentiles_to_labels(kth_percentiles);
for (const auto kth_label : kth_labels) {
metric_snapshot::attr_map labels({{"p", kth_label}});
labels.insert(attrs.begin(), attrs.end());
metric_snapshot snapshot(m->prototype()->name(), m->prototype()->type(), value, std::move(labels));
expected_metrics[m->prototype()->name()][snapshot.encode_attributes()] = std::move(snapshot);
}
}

TEST_F(metrics_test, metric_data_sink)
{
const std::string target_attr_key("_DEDICATED_FOR_DATA_SINK_TEST");
@@ -1035,12 +1058,22 @@ TEST_F(metrics_test, metric_data_sink)
auto my_server_entity = METRIC_ENTITY_my_server.instantiate(test.entity_id, attrs);

auto my_gauge_int64 = METRIC_test_gauge_int64.instantiate(my_server_entity);
my_gauge_int64.set(test.gauge_int64_value);
my_gauge_int64->set(test.gauge_int64_value);
build_expected_snapshots(my_gauge_int64.get(), static_cast<metric_snapshot::value_type>(test.gauge_int64_value), test.entity_attrs, expected_metrics);

auto my_gauge_double = METRIC_test_gauge_double.instantiate(my_server_entity);
my_gauge_double.set(test.gauge_double_value);
my_gauge_double->set(test.gauge_double_value);
build_expected_snapshots(my_gauge_double.get(), static_cast<metric_snapshot::value_type>(test.gauge_double_value), test.entity_attrs, expected_metrics);

auto my_counter = METRIC_test_counter.instantiate(my_server_entity);
my_counter->increment_by(test.counter_increase);
build_expected_snapshots(my_counter.get(), static_cast<metric_snapshot::value_type>(test.counter_increase), test.entity_attrs, static_cast<metric_snapshot::value_type>(test.counter_increase), expected_counters);

auto my_concurrent_counter = METRIC_test_concurrent_counter.instantiate(my_server_entity);
my_concurrent_counter->increment_by(test.counter_increase);
build_expected_snapshots(my_concurrent_counter.get(), static_cast<metric_snapshot::value_type>(test.counter_increase), test.entity_attrs, static_cast<metric_snapshot::value_type>(test.counter_increase), expected_counters);

auto my_metric = prototype.instantiate(my_entity, interval_ms, kth_percentiles, sample_size);
}

check_snapshots(expected_metrics, expected_counters);