From a402769956f6c6c6b1ef0122c495977763e65722 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Fri, 26 Aug 2016 12:46:56 +0100 Subject: [PATCH 1/2] Add commit info to night-rally output --- benchmarks/result.csv | 56 ++++++++++++++++++++++++++++++++++++++ benchmarks/result.csv.meta | 2 ++ night_rally.py | 35 ++++++++++++++++++++++++ tests.py | 6 ++++ 4 files changed, 99 insertions(+) create mode 100644 benchmarks/result.csv create mode 100644 benchmarks/result.csv.meta diff --git a/benchmarks/result.csv b/benchmarks/result.csv new file mode 100644 index 0000000000000..32305b467cf59 --- /dev/null +++ b/benchmarks/result.csv @@ -0,0 +1,56 @@ +Metric,Value +Min Indexing Throughput [docs/s],38556.0 +Median Indexing Throughput [docs/s],44566.58333333333 +Max Indexing Throughput [docs/s],48244.0 +Indexing time [min],30.72458333333333 +Merge time [min],7.18975 +Refresh time [min],2.3497833333333333 +Flush time [min],1.2546 +Merge throttle time [min],1.3034166666666667 +Query latency default (90.0 percentile) [ms],54.36155014038086 +Query latency default (99.0 percentile) [ms],73.54597908020052 +Query latency default (100.0 percentile) [ms],135.98109436035156 +Query latency term (90.0 percentile) [ms],1.802869987487793 +Query latency term (99.0 percentile) [ms],2.0635323977470406 +Query latency term (100.0 percentile) [ms],2.1558499336242676 +Query latency phrase (90.0 percentile) [ms],2.4050144672393805 +Query latency phrase (99.0 percentile) [ms],2.8553510928154013 +Query latency phrase (100.0 percentile) [ms],3.282254934310913 +Query latency country_agg_uncached (90.0 percentile) [ms],263.99256286621096 +Query latency country_agg_uncached (99.0 percentile) [ms],301.94572845458987 +Query latency country_agg_uncached (100.0 percentile) [ms],302.1258850097656 +Query latency country_agg_cached (90.0 percentile) [ms],2.2271822214126584 +Query latency country_agg_cached (99.0 percentile) [ms],5.126538918019618 +Query latency country_agg_cached (100.0 percentile) [ms],252.0946044921875 +Query latency scroll (90.0 percentile) [ms],23.321646308898927 +Query latency scroll (99.0 percentile) [ms],24.782226085662845 +Query latency scroll (100.0 percentile) [ms],25.958951950073242 +Query latency expression (90.0 percentile) [ms],381.54732055664067 +Query latency expression (99.0 percentile) [ms],385.22566650390627 +Query latency expression (100.0 percentile) [ms],385.9678039550781 +Query latency painless_static (90.0 percentile) [ms],540.9335327148438 +Query latency painless_static (99.0 percentile) [ms],546.2719622802734 +Query latency painless_static (100.0 percentile) [ms],549.9568481445312 +Query latency painless_dynamic (90.0 percentile) [ms],525.440496826172 +Query latency painless_dynamic (99.0 percentile) [ms],556.6867095947267 +Query latency painless_dynamic (100.0 percentile) [ms],575.1890869140625 +Median CPU usage (index) [%],957.1499938964844 +Median CPU usage (stats) [%],150.35000228881836 +Median CPU usage (search) [%],352.5000010172526 +Total Young Gen GC [s],11.273 +Total Old Gen GC [s],1.535 +Index size [GB],3.279082341119647 +Totally written [GB],16.861427307128906 +Heap used for segments [MB],21.234503746032715 +Heap used for doc values [MB],0.10956192016601562 +Heap used for terms [MB],19.60907554626465 +Heap used for norms [MB],0.07666015625 +Heap used for points [MB],0.7714815139770508 +Heap used for stored fields [MB],0.667724609375 +Segment count,97 +Indices Stats(90.0 percentile) [ms],3.9261399745941166 +Indices Stats(99.0 percentile) [ms],4.425495061874392 +Indices Stats(100.0 percentile) [ms],4.847516059875488 +Nodes Stats(90.0 percentile) [ms],4.01015009880066 +Nodes Stats(99.0 percentile) [ms],4.911499872207657 +Nodes Stats(100.0 percentile) [ms],7.960392951965332 diff --git a/benchmarks/result.csv.meta b/benchmarks/result.csv.meta new file mode 100644 index 0000000000000..ac324154c984e --- /dev/null +++ b/benchmarks/result.csv.meta @@ -0,0 +1,2 @@ +Name,Value +Elasticsearch source revision,1b1a1ac diff --git a/night_rally.py b/night_rally.py index 464a2384cd368..8e27e0f1ae004 100644 --- a/night_rally.py +++ b/night_rally.py @@ -185,6 +185,10 @@ def v(d, k): "Merge time \(.*": "merge_time_parts" } +META_METRICS_TO_KEY = { + "Elasticsearch source revision": "source_revision" +} + def is_default_setup(default_setup, current_setup): current_challenge, current_car = current_setup @@ -218,6 +222,11 @@ def key_for(metric_pattern, metric_key, metric_name, op_name): return metric_key return None +def meta_key_for(metric_pattern, metric_key, metric_name): + if re.match(metric_pattern, metric_name): + return metric_key + return None + def extract_metrics(source_report): metrics = {} @@ -237,6 +246,22 @@ def extract_metrics(source_report): metrics[final_key] = metric_value return metrics +def extract_meta_metrics(source_meta_report): + meta_metrics = {} + for row in csv.reader(source_meta_report): + for metric_pattern, metric_key in META_METRICS_TO_KEY.items(): + metric_name = row[0] + metric_value = row[1] + final_key = meta_key_for(metric_pattern, metric_key, metric_name) + if final_key: + if is_multi_valued(final_key): + if final_key not in meta_metrics: + meta_metrics[final_key] = [] + meta_metrics[final_key].append(metric_value) + else: + meta_metrics[final_key] = metric_value + return meta_metrics + def report(effective_start_date, tracks, default_setup_per_track): """ @@ -263,6 +288,7 @@ def report(effective_start_date, tracks, default_setup_per_track): challenge, car = setup current_is_default = is_default_setup(default_setup_per_track[track], setup) report_path = "%s/%s/rally/%s/%s/%s/%s/report.csv" % (root_dir, report_root_dir, timestamp, track, challenge, car) + meta_report_path = "%s/%s/rally/%s/%s/%s/%s/report.csv.meta" % (root_dir, report_root_dir, timestamp, track, challenge, car) if not os.path.isfile(report_path): logger.warn("[%s] does not exist. Skipping track [%s], challenge [%s], car [%s]." @@ -326,6 +352,15 @@ def report(effective_start_date, tracks, default_setup_per_track): with open("%s/merge_parts.csv" % output_report_path, "a") as f: f.write("%s,%s\n" % (report_timestamp, ",".join(metrics["merge_time_parts"]))) + + + with open(meta_report_path) as csvfile: + meta_metrics = extract_meta_metrics(csvfile) + + if "source_revision" in metrics: + with open("%s/source_revision.csv" % output_report_path, "a") as f: + f.write("%s,%s\n" % (report_timestamp, ",".join(metrics["source_revision"]))) + if len(segment_count_metrics) > 0: with open("%s/segment_counts.csv" % output_report_path, "a") as f: f.write("%s,%s\n" % (report_timestamp, ",".join(segment_count_metrics))) diff --git a/tests.py b/tests.py index dbdf32b7e5a19..2cc499b3f9bd9 100644 --- a/tests.py +++ b/tests.py @@ -128,6 +128,8 @@ 99.0th percentile service time,painless_static,13.627798499919663,ms 100th percentile service time,painless_static,13.649789999817585,ms""" +EXAMPLE_META_REPORT = """Name,Value +Elasticsearch source revision,247cafe""" class RecordingSystemCall: def __init__(self, return_value): @@ -250,6 +252,10 @@ def test_extract_metrics(self): self.assertEqual("8.401418720027323", metrics["latency_nodes_stats_p99"]) self.assertEqual(["260", "362", "327", "7", "306", "553", "12", "13"], metrics["query_latency_p99"]) + def test_extract_meta_metrics(self): + metrics = night_rally.extract_meta_metrics(EXAMPLE_META_REPORT.split("\n")) + self.assertEqual("247cafe", metrics["source_revision"]) + if __name__ == '__main__': unittest.main() From b51663f416d256cf838c8046fca8c9c890a69a76 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Fri, 26 Aug 2016 17:01:30 +0100 Subject: [PATCH 2/2] add click data point to compare commits on github to benchmark charts --- benchmarks/result.csv | 56 ---------------------------- benchmarks/result.csv.meta | 2 - external/pages/assets/js/charts.js | 53 +++++++++++++++++++------- external/pages/geonames/index.html | 8 +++- external/pages/geopoint/index.html | 6 ++- external/pages/percolator/index.html | 6 ++- external/pages/pmc/index.html | 6 ++- 7 files changed, 61 insertions(+), 76 deletions(-) delete mode 100644 benchmarks/result.csv delete mode 100644 benchmarks/result.csv.meta diff --git a/benchmarks/result.csv b/benchmarks/result.csv deleted file mode 100644 index 32305b467cf59..0000000000000 --- a/benchmarks/result.csv +++ /dev/null @@ -1,56 +0,0 @@ -Metric,Value -Min Indexing Throughput [docs/s],38556.0 -Median Indexing Throughput [docs/s],44566.58333333333 -Max Indexing Throughput [docs/s],48244.0 -Indexing time [min],30.72458333333333 -Merge time [min],7.18975 -Refresh time [min],2.3497833333333333 -Flush time [min],1.2546 -Merge throttle time [min],1.3034166666666667 -Query latency default (90.0 percentile) [ms],54.36155014038086 -Query latency default (99.0 percentile) [ms],73.54597908020052 -Query latency default (100.0 percentile) [ms],135.98109436035156 -Query latency term (90.0 percentile) [ms],1.802869987487793 -Query latency term (99.0 percentile) [ms],2.0635323977470406 -Query latency term (100.0 percentile) [ms],2.1558499336242676 -Query latency phrase (90.0 percentile) [ms],2.4050144672393805 -Query latency phrase (99.0 percentile) [ms],2.8553510928154013 -Query latency phrase (100.0 percentile) [ms],3.282254934310913 -Query latency country_agg_uncached (90.0 percentile) [ms],263.99256286621096 -Query latency country_agg_uncached (99.0 percentile) [ms],301.94572845458987 -Query latency country_agg_uncached (100.0 percentile) [ms],302.1258850097656 -Query latency country_agg_cached (90.0 percentile) [ms],2.2271822214126584 -Query latency country_agg_cached (99.0 percentile) [ms],5.126538918019618 -Query latency country_agg_cached (100.0 percentile) [ms],252.0946044921875 -Query latency scroll (90.0 percentile) [ms],23.321646308898927 -Query latency scroll (99.0 percentile) [ms],24.782226085662845 -Query latency scroll (100.0 percentile) [ms],25.958951950073242 -Query latency expression (90.0 percentile) [ms],381.54732055664067 -Query latency expression (99.0 percentile) [ms],385.22566650390627 -Query latency expression (100.0 percentile) [ms],385.9678039550781 -Query latency painless_static (90.0 percentile) [ms],540.9335327148438 -Query latency painless_static (99.0 percentile) [ms],546.2719622802734 -Query latency painless_static (100.0 percentile) [ms],549.9568481445312 -Query latency painless_dynamic (90.0 percentile) [ms],525.440496826172 -Query latency painless_dynamic (99.0 percentile) [ms],556.6867095947267 -Query latency painless_dynamic (100.0 percentile) [ms],575.1890869140625 -Median CPU usage (index) [%],957.1499938964844 -Median CPU usage (stats) [%],150.35000228881836 -Median CPU usage (search) [%],352.5000010172526 -Total Young Gen GC [s],11.273 -Total Old Gen GC [s],1.535 -Index size [GB],3.279082341119647 -Totally written [GB],16.861427307128906 -Heap used for segments [MB],21.234503746032715 -Heap used for doc values [MB],0.10956192016601562 -Heap used for terms [MB],19.60907554626465 -Heap used for norms [MB],0.07666015625 -Heap used for points [MB],0.7714815139770508 -Heap used for stored fields [MB],0.667724609375 -Segment count,97 -Indices Stats(90.0 percentile) [ms],3.9261399745941166 -Indices Stats(99.0 percentile) [ms],4.425495061874392 -Indices Stats(100.0 percentile) [ms],4.847516059875488 -Nodes Stats(90.0 percentile) [ms],4.01015009880066 -Nodes Stats(99.0 percentile) [ms],4.911499872207657 -Nodes Stats(100.0 percentile) [ms],7.960392951965332 diff --git a/benchmarks/result.csv.meta b/benchmarks/result.csv.meta deleted file mode 100644 index ac324154c984e..0000000000000 --- a/benchmarks/result.csv.meta +++ /dev/null @@ -1,2 +0,0 @@ -Name,Value -Elasticsearch source revision,1b1a1ac diff --git a/external/pages/assets/js/charts.js b/external/pages/assets/js/charts.js index 1a1737c4e6691..888187d55bc1d 100644 --- a/external/pages/assets/js/charts.js +++ b/external/pages/assets/js/charts.js @@ -1,5 +1,5 @@ var chartingLib = { - createOpts: function(title, labelsDiv, yLabel) { + createOpts: function(title, labelsDiv, yLabel, commits) { return { "title": title, "colors": ["#00BFB3", "#FED10A", "#0078A0", "#DF4998", "#93C90E", "#00A9E5", "#222", "#AAA", "#777"], "axisLabelColor": "#555", @@ -15,24 +15,51 @@ var chartingLib = { "legend": "always", "drawPoints": true, "pointSize": 3, - "gridLineColor": "#BBB" + "gridLineColor": "#BBB", + pointClickCallback: function(e, p) { + var commitInfo = commits[p.xval]; + if (commitInfo) { + var url = "https://github.com/elastic/elasticsearch/compare/" + commitInfo.prev_commit + "..." + commitInfo.this_commit; + window.open(url); + } + } //"strokePattern": [5, 10] } }, renderChart: function(elementId, title, yLabel, annotationSource) { - var g = new Dygraph( - document.getElementById("chart_" + elementId), - elementId + ".csv", - this.createOpts("" + title + "", "chart_" + elementId + "_labels", yLabel) - ); - g.ready(function() { - var annotationSource = elementId + '_annotations.json' - $.getJSON(annotationSource, function(json) { - g.setAnnotations(json); - }); + var commits = {}; + var g = new Dygraph( + document.getElementById("chart_" + elementId), + elementId + ".csv", + this.createOpts("" + title + "", "chart_" + elementId + "_labels", yLabel, commits) + ); + g.ready(function() { + var annotationSource = elementId + '_annotations.json' + $.getJSON(annotationSource, function(json) { + g.setAnnotations(json); }); - return g; + $.ajax({ + type: 'GET', + url: 'source_revision.csv', + dataType: 'text', + success: function(allText) { + var allTextLines = allText.split(/\r\n|\n/); + + var previousCommit = null; + for (var i=1; iOverview Kibana dashboard.

- +

Results

@@ -126,6 +126,10 @@

Results

Hint Click and drag to zoom. Double-click to zoom out.
+
@@ -195,7 +199,7 @@

Results

- +
diff --git a/external/pages/geopoint/index.html b/external/pages/geopoint/index.html index 0d92d3385d30b..9cefc793ab7cf 100644 --- a/external/pages/geopoint/index.html +++ b/external/pages/geopoint/index.html @@ -121,6 +121,10 @@

Results

Hint Click and drag to zoom. Double-click to zoom out. + @@ -180,7 +184,7 @@

Results

- +
diff --git a/external/pages/percolator/index.html b/external/pages/percolator/index.html index 21a957b35f0dc..3c8bb3a17f988 100644 --- a/external/pages/percolator/index.html +++ b/external/pages/percolator/index.html @@ -115,6 +115,10 @@

Results

Hint Click and drag to zoom. Double-click to zoom out. + @@ -174,7 +178,7 @@

Results

- +
diff --git a/external/pages/pmc/index.html b/external/pages/pmc/index.html index 3bbb2be81b231..f23cd41d51714 100644 --- a/external/pages/pmc/index.html +++ b/external/pages/pmc/index.html @@ -121,6 +121,10 @@

Results

Hint Click and drag to zoom. Double-click to zoom out. + @@ -180,7 +184,7 @@

Results

- +