From dfa9c0d267287670e6a1989cc5b4473c28627801 Mon Sep 17 00:00:00 2001 From: PMazarovich <57707673+PMazarovich@users.noreply.github.com> Date: Tue, 26 Jan 2021 12:00:30 +0300 Subject: [PATCH] Logstash+Kibana improvements (#2531) --- CHANGELOG.md | 1 + components/analytics/docker-compose.analytics.yml | 4 ++++ components/analytics/kibana/export.json | 11 +++++++++-- components/analytics/logstash/logstash.conf | 10 ++++++++-- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c37b869bc63f..bec9fa89781c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added auto inference of url schema from host in CLI, if provided () - Track frames in skips between annotation is presented in MOT and MOTS formats are marked `outside` () - UI packages installation with `npm ci` instead of `npm install` () +- Logstash is improved for using with configurable elasticsearch outputs () ### Removed diff --git a/components/analytics/docker-compose.analytics.yml b/components/analytics/docker-compose.analytics.yml index c7a2296330cd..e7890ab16e3d 100644 --- a/components/analytics/docker-compose.analytics.yml +++ b/components/analytics/docker-compose.analytics.yml @@ -69,6 +69,10 @@ services: ELK_VERSION: 6.4.0 http_proxy: ${http_proxy} https_proxy: ${https_proxy} + environment: + LOGSTASH_OUTPUT_HOST: elasticsearch:9200 + LOGSTASH_OUTPUT_USER: + LOGSTASH_OUTPUT_PASS: depends_on: ['cvat_elasticsearch'] restart: always diff --git a/components/analytics/kibana/export.json b/components/analytics/kibana/export.json index 5c920e9ee965..7ba8d6372a96 100644 --- a/components/analytics/kibana/export.json +++ b/components/analytics/kibana/export.json @@ -58,14 +58,21 @@ "_type": "visualization", "_source": { "title": "Timeline for exceptions", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"1-bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"filters\"},\"customLabel\":\"Exceptions\",\"customMetric\":{\"enabled\":true,\"id\":\"1-metric\",\"params\":{\"customLabel\":\"Exceptions\"},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Time\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"title\":\"Timeline for exceptions\",\"type\":\"histogram\"}", + "visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\",\"language\":\"lucene\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":false,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Exceptions\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Exceptions\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"orderBucketsBySum\":false}}", "uiStateJSON": "{}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, + "references": [ + { + "id": "ec510550-c238-11e8-8e1b-758ef07f6de8", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "_meta": { "savedObjectVersion": 2 } diff --git a/components/analytics/logstash/logstash.conf b/components/analytics/logstash/logstash.conf index cbf893f6020e..02b3a5f29f61 100644 --- a/components/analytics/logstash/logstash.conf +++ b/components/analytics/logstash/logstash.conf @@ -106,13 +106,19 @@ output { if [type] == "client" { elasticsearch { - hosts => ["elasticsearch:9200"] + hosts => ["${LOGSTASH_OUTPUT_HOST}"] index => "cvat.client" + user => "${LOGSTASH_OUTPUT_USER:}" + password => "${LOGSTASH_OUTPUT_PASS:}" + manage_template => false } } else if [type] == "server" { elasticsearch { - hosts => ["elasticsearch:9200"] + hosts => ["${LOGSTASH_OUTPUT_HOST}"] index => "cvat.server" + user => "${LOGSTASH_OUTPUT_USER:}" + password => "${LOGSTASH_OUTPUT_PASS:}" + manage_template => false } } }