From 8c2b883cc9872b8901e24b9051a8a0e2f4bb2912 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Wed, 19 Sep 2018 11:22:42 +0300 Subject: [PATCH 01/19] First version of ELK stack for analytics. --- analytics/.env | 7 ++++ analytics/docker-compose.yml | 46 +++++++++++++++++++++++ analytics/elasticsearch/Dockerfile | 3 ++ analytics/elasticsearch/elasticsearch.yml | 2 + analytics/kibana/Dockerfile | 5 +++ analytics/kibana/kibana.yml | 3 ++ analytics/logstash/Dockerfile | 7 ++++ analytics/logstash/logstash.conf | 26 +++++++++++++ 8 files changed, 99 insertions(+) create mode 100644 analytics/.env create mode 100644 analytics/docker-compose.yml create mode 100644 analytics/elasticsearch/Dockerfile create mode 100644 analytics/elasticsearch/elasticsearch.yml create mode 100644 analytics/kibana/Dockerfile create mode 100644 analytics/kibana/kibana.yml create mode 100644 analytics/logstash/Dockerfile create mode 100644 analytics/logstash/logstash.conf diff --git a/analytics/.env b/analytics/.env new file mode 100644 index 000000000000..2555566d250d --- /dev/null +++ b/analytics/.env @@ -0,0 +1,7 @@ +LOGSTASH_PORT=5000 +ELASTICSEARCH_HOST=cvat_elasticsearch +ELASTICSEARCH_PORT=9200 +KIBANA_HOST=cvat_kibana +KIBANA_PORT=5601 +http_proxy= +https_proxy= diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml new file mode 100644 index 000000000000..c4859d5174fe --- /dev/null +++ b/analytics/docker-compose.yml @@ -0,0 +1,46 @@ +version: '2.3' +services: + cvat_elasticsearch: + container_name: cvat_elasticsearch + env_file: + - .env + build: + context: ./analytics/elasticsearch + args: + ELK_VERSION: 6.4.0 + image: cvat_elasticsearch + ports: + - "${ELASTICSEARCH_PORT}:9200" + restart: always + + cvat_kibana: + container_name: cvat_kibana + env_file: + - .env + build: + context: ./analytics/kibana + args: + ELK_VERSION: 6.4.0 + image: cvat_kibana + ports: + - "${KIBANA_PORT}:5601" + restart: always + + cvat_logstash: + container_name: cvat_logstash + env_file: + - .env + build: + context: ./analytics/logstash + args: + ELK_VERSION: 6.4.0 + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + image: cvat_logstash + ports: + - "${LOGSTASH_PORT}:5000" + restart: always + + cvat: + environment: + DJANGO_LOG_SERVER_URL: "${LOGSTASH_PORT}:5000" diff --git a/analytics/elasticsearch/Dockerfile b/analytics/elasticsearch/Dockerfile new file mode 100644 index 000000000000..e015784449af --- /dev/null +++ b/analytics/elasticsearch/Dockerfile @@ -0,0 +1,3 @@ +ARG ELK_VERSION +FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION} +COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/ diff --git a/analytics/elasticsearch/elasticsearch.yml b/analytics/elasticsearch/elasticsearch.yml new file mode 100644 index 000000000000..c7c90a58bab7 --- /dev/null +++ b/analytics/elasticsearch/elasticsearch.yml @@ -0,0 +1,2 @@ +http.host: 0.0.0.0 +script.painless.regex.enabled: true diff --git a/analytics/kibana/Dockerfile b/analytics/kibana/Dockerfile new file mode 100644 index 000000000000..f7982dffc29d --- /dev/null +++ b/analytics/kibana/Dockerfile @@ -0,0 +1,5 @@ +ARG ELK_VERSION +FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION} +COPY kibana.yml /usr/share/kibana/config/ + + diff --git a/analytics/kibana/kibana.yml b/analytics/kibana/kibana.yml new file mode 100644 index 000000000000..7389d05d1ba4 --- /dev/null +++ b/analytics/kibana/kibana.yml @@ -0,0 +1,3 @@ +server.host: 0.0.0.0 +elasticsearch.url: http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT} +elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ] diff --git a/analytics/logstash/Dockerfile b/analytics/logstash/Dockerfile new file mode 100644 index 000000000000..f7289a69bd6c --- /dev/null +++ b/analytics/logstash/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION +FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} + +RUN logstash-plugin install logstash-input-http + +COPY logstash.conf ~/ +CMD ["-f", "~/logstash.conf"] diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf new file mode 100644 index 000000000000..367e7a49b7fd --- /dev/null +++ b/analytics/logstash/logstash.conf @@ -0,0 +1,26 @@ +input { + http { + port => ${LOGSTASH_PORT} + response_headers => { + "Access-Control-Allow-Origin" => "*" + "Access-Control-Allow-Headers" => "Content-Type" + } + } +} + +filter { + if [headers][request_method] == "OPTIONS" { + drop { } + } +} + +output { + stdout { + codec => rubydebug + } + + elasticsearch { + hosts => "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}" + index => "cvat-client" + } +} From da7a37dcc5a5d84c84f9ce2952080909cf632fe6 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Wed, 19 Sep 2018 19:07:59 +0300 Subject: [PATCH 02/19] Fixed LOGSERVER URL --- analytics/.env | 7 ------- analytics/docker-compose.yml | 5 +++-- analytics/logstash/Dockerfile | 4 +--- 3 files changed, 4 insertions(+), 12 deletions(-) delete mode 100644 analytics/.env diff --git a/analytics/.env b/analytics/.env deleted file mode 100644 index 2555566d250d..000000000000 --- a/analytics/.env +++ /dev/null @@ -1,7 +0,0 @@ -LOGSTASH_PORT=5000 -ELASTICSEARCH_HOST=cvat_elasticsearch -ELASTICSEARCH_PORT=9200 -KIBANA_HOST=cvat_kibana -KIBANA_PORT=5601 -http_proxy= -https_proxy= diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index c4859d5174fe..2c085b510459 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -36,11 +36,12 @@ services: ELK_VERSION: 6.4.0 http_proxy: ${http_proxy} https_proxy: ${https_proxy} + no_proxy: ${no_proxy} image: cvat_logstash ports: - - "${LOGSTASH_PORT}:5000" + - "${LOGSTASH_PORT}:5044" restart: always cvat: environment: - DJANGO_LOG_SERVER_URL: "${LOGSTASH_PORT}:5000" + DJANGO_LOG_SERVER_URL: "http://${LOGSTASH_HOST}:${LOGSTASH_PORT}" diff --git a/analytics/logstash/Dockerfile b/analytics/logstash/Dockerfile index f7289a69bd6c..3088c6a8b6fe 100644 --- a/analytics/logstash/Dockerfile +++ b/analytics/logstash/Dockerfile @@ -1,7 +1,5 @@ ARG ELK_VERSION FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} - RUN logstash-plugin install logstash-input-http -COPY logstash.conf ~/ -CMD ["-f", "~/logstash.conf"] +COPY logstash.conf /usr/share/logstash/ From a964cafbe9baebfded0ea275643b1fd97ec118ff Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Wed, 26 Sep 2018 18:21:32 +0300 Subject: [PATCH 03/19] Analytics - resolve problems Two major problems: - proxy configuration (no_proxy should contain logstash, elasticsearch for corresponding containers) - aliases for hosts (logstash doesn't work with hostnames which contain '_') Also decided don't use .env file for docker-compose. --- analytics/docker-compose.yml | 32 ++++++++++++++++++++------------ analytics/kibana/kibana.yml | 2 +- analytics/logstash/Dockerfile | 2 +- analytics/logstash/logstash.conf | 4 ++-- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index 2c085b510459..261df4818988 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -2,46 +2,54 @@ version: '2.3' services: cvat_elasticsearch: container_name: cvat_elasticsearch - env_file: - - .env + networks: + default: + aliases: + - elasticsearch build: context: ./analytics/elasticsearch args: ELK_VERSION: 6.4.0 image: cvat_elasticsearch - ports: - - "${ELASTICSEARCH_PORT}:9200" restart: always cvat_kibana: container_name: cvat_kibana - env_file: - - .env + networks: + default: + aliases: + - kibana build: context: ./analytics/kibana args: ELK_VERSION: 6.4.0 image: cvat_kibana ports: - - "${KIBANA_PORT}:5601" + - "5601:5601" restart: always + environment: + no_proxy: elasticsearch,${no_proxy} cvat_logstash: container_name: cvat_logstash - env_file: - - .env + networks: + default: + aliases: + - logstash build: context: ./analytics/logstash args: ELK_VERSION: 6.4.0 http_proxy: ${http_proxy} https_proxy: ${https_proxy} - no_proxy: ${no_proxy} image: cvat_logstash ports: - - "${LOGSTASH_PORT}:5044" + - "5000:5000" restart: always + environment: + no_proxy: elasticsearch,${no_proxy} cvat: environment: - DJANGO_LOG_SERVER_URL: "http://${LOGSTASH_HOST}:${LOGSTASH_PORT}" + DJANGO_LOG_SERVER_URL: "http://logstash:5000" + no_proxy: logstash,${no_proxy} diff --git a/analytics/kibana/kibana.yml b/analytics/kibana/kibana.yml index 7389d05d1ba4..30f881e93000 100644 --- a/analytics/kibana/kibana.yml +++ b/analytics/kibana/kibana.yml @@ -1,3 +1,3 @@ server.host: 0.0.0.0 -elasticsearch.url: http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT} +elasticsearch.url: http://elasticsearch:9200 elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ] diff --git a/analytics/logstash/Dockerfile b/analytics/logstash/Dockerfile index 3088c6a8b6fe..25d80829b3aa 100644 --- a/analytics/logstash/Dockerfile +++ b/analytics/logstash/Dockerfile @@ -2,4 +2,4 @@ ARG ELK_VERSION FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} RUN logstash-plugin install logstash-input-http -COPY logstash.conf /usr/share/logstash/ +COPY logstash.conf /usr/share/logstash/pipeline diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf index 367e7a49b7fd..285398c60036 100644 --- a/analytics/logstash/logstash.conf +++ b/analytics/logstash/logstash.conf @@ -1,6 +1,6 @@ input { http { - port => ${LOGSTASH_PORT} + port => 5000 response_headers => { "Access-Control-Allow-Origin" => "*" "Access-Control-Allow-Headers" => "Content-Type" @@ -20,7 +20,7 @@ output { } elasticsearch { - hosts => "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}" + hosts => ["elasticsearch:9200"] index => "cvat-client" } } From 01139b3dadd3b8206d24f1dcdc70a3ceb8879fb0 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 27 Sep 2018 18:30:44 +0300 Subject: [PATCH 04/19] Add a temlpate of dashboards for kibana --- analytics/docker-compose.yml | 14 +-- analytics/kibana/template/cvat.json | 148 ++++++++++++++++++++++++++++ analytics/logstash/Dockerfile | 6 +- analytics/logstash/logstash.conf | 34 +++++++ 4 files changed, 190 insertions(+), 12 deletions(-) create mode 100644 analytics/kibana/template/cvat.json diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index 261df4818988..b65e67ba4464 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -2,6 +2,7 @@ version: '2.3' services: cvat_elasticsearch: container_name: cvat_elasticsearch + image: cvat_elasticsearch networks: default: aliases: @@ -10,11 +11,11 @@ services: context: ./analytics/elasticsearch args: ELK_VERSION: 6.4.0 - image: cvat_elasticsearch restart: always cvat_kibana: container_name: cvat_kibana + image: cvat_kibana networks: default: aliases: @@ -23,15 +24,13 @@ services: context: ./analytics/kibana args: ELK_VERSION: 6.4.0 - image: cvat_kibana ports: - "5601:5601" restart: always - environment: - no_proxy: elasticsearch,${no_proxy} cvat_logstash: container_name: cvat_logstash + image: cvat_logstash networks: default: aliases: @@ -40,14 +39,9 @@ services: context: ./analytics/logstash args: ELK_VERSION: 6.4.0 - http_proxy: ${http_proxy} + http_proxy: ${http_proxy} https_proxy: ${https_proxy} - image: cvat_logstash - ports: - - "5000:5000" restart: always - environment: - no_proxy: elasticsearch,${no_proxy} cvat: environment: diff --git a/analytics/kibana/template/cvat.json b/analytics/kibana/template/cvat.json new file mode 100644 index 000000000000..ccfb29d743a3 --- /dev/null +++ b/analytics/kibana/template/cvat.json @@ -0,0 +1,148 @@ +[ + { + "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", + "_source": { + "title": "Monitoring", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", + "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + } + }, + { + "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", + "_source": { + "title": "Managment", + "hits": 0, + "description": "", + "panelsJSON": "[{\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":33,\"i\":\"1\"},\"version\":\"6.4.0\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":33,\"i\":\"2\"},\"version\":\"6.4.0\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":20,\"i\":\"3\"},\"version\":\"6.4.0\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}}]", + "optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8", + "_type": "index-pattern", + "_source": { + "title": "cvat*", + "timeFieldName": "@timestamp", + "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}" + } + }, + { + "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8", + "_type": "search", + "_source": { + "title": "Exceptions table", + "description": "", + "hits": 0, + "columns": [ + "event" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}" + } + } + }, + { + "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Exceptions timeline", + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"1-bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"filters\"},\"customLabel\":\"Exceptions\",\"customMetric\":{\"enabled\":true,\"id\":\"1-metric\",\"params\":{\"customLabel\":\"Exceptions\"},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Time\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"title\":\"Exceptions timeline\",\"type\":\"histogram\"}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Users", + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"User\",\"field\":\"userid.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"3-bucket\",\"params\":{\"customInterval\":\"2h\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"date_histogram\"},\"customLabel\":\"Activity\",\"customMetric\":{\"enabled\":true,\"id\":\"3-metric\",\"params\":{},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Working Time (h)\",\"field\":\"working time\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Users\",\"type\":\"table\"}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + } + }, + { + "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Events duration", + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"4\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"min\"},{\"enabled\":true,\"id\":\"5\",\"params\":{\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"max\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events duration\",\"type\":\"table\"}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}" + } + } + }, + { + "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Working Day", + "visState": "{\"title\":\"Working Day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + }, + { + "_id": "65918380-c244-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Events count", + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events count\",\"type\":\"table\"}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + } + }, + { + "_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Users Activity", + "visState": "{\"title\":\"Users Activity\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + } + } +] \ No newline at end of file diff --git a/analytics/logstash/Dockerfile b/analytics/logstash/Dockerfile index 25d80829b3aa..ad012ccfc8fa 100644 --- a/analytics/logstash/Dockerfile +++ b/analytics/logstash/Dockerfile @@ -1,5 +1,7 @@ ARG ELK_VERSION FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} -RUN logstash-plugin install logstash-input-http +RUN logstash-plugin install logstash-input-http logstash-filter-aggregate \ + logstash-filter-prune logstash-output-email -COPY logstash.conf /usr/share/logstash/pipeline +COPY logstash.conf /usr/share/logstash/pipeline/ +EXPOSE 5000 diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf index 285398c60036..16362184cafb 100644 --- a/analytics/logstash/logstash.conf +++ b/analytics/logstash/logstash.conf @@ -12,6 +12,40 @@ filter { if [headers][request_method] == "OPTIONS" { drop { } } + + prune { + blacklist_names => ["headers", "host"] + } + + if [event] == "Send exception" { + aggregate { + task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}" + code => " + require 'time' + + map['userid'] ||= event.get('userid'); + map['application'] ||= event.get('application'); + map['message'] ||= event.get('message'); + map['filename'] ||= event.get('filename'); + map['line'] ||= event.get('line'); + map['task'] ||= event.get('task'); + + map['error_count'] ||= 0; + map['error_count'] += 1; + + map['aggregated_message'] ||= ''; + time = Time.strptime(event.get('timestamp').to_s,'%Q').localtime('+03:00') + map['aggregated_message'] += time.to_s + '\n' + event.get('stack') + '\n\n\n';" + + timeout => 3600 + timeout_tags => ['send_email_notification'] + push_map_as_event_on_timeout => true + } + } + + date { + match => ["timestamp", "UNIX", "UNIX_MS"] + } } output { From cea435a14e5e4d9f16ca1697bb4f4141a34a7d71 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Fri, 28 Sep 2018 19:50:56 +0300 Subject: [PATCH 05/19] Minor refactoring --- analytics/kibana/{template => config}/cvat.json | 0 docker-compose.yml | 10 ++++++++++ supervisord.conf | 11 +++++++---- 3 files changed, 17 insertions(+), 4 deletions(-) rename analytics/kibana/{template => config}/cvat.json (100%) diff --git a/analytics/kibana/template/cvat.json b/analytics/kibana/config/cvat.json similarity index 100% rename from analytics/kibana/template/cvat.json rename to analytics/kibana/config/cvat.json diff --git a/docker-compose.yml b/docker-compose.yml index 490684f0f63b..6ebae20e7478 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,10 @@ services: cvat_db: container_name: cvat_db image: postgres:10.3-alpine + networks: + default: + aliases: + - db restart: always environment: POSTGRES_USER: root @@ -19,6 +23,12 @@ services: cvat_redis: container_name: cvat_redis image: redis:4.0.5-alpine + container_name: cvat_db + image: postgres:10.3-alpine + networks: + default: + aliases: + - redis restart: always cvat: diff --git a/supervisord.conf b/supervisord.conf index 70c3cf8dee38..87020e5b8a09 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -18,16 +18,19 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker_default] -command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic "/usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" +command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic \ + "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" numprocs=2 process_name=rqworker_default_%(process_num)s [program:rqworker_low] -command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic "/usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" +command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ + "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" numprocs=1 [program:runserver] -command=%(ENV_HOME)s/wait-for-it.sh cvat_db:5432 -t 0 -- bash -ic "/usr/bin/python3 ~/manage.py migrate && \ - exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ +command=%(ENV_HOME)s/wait-for-it.sh db:5432 -t 0 -- bash -ic \ + "/usr/bin/python3 ~/manage.py migrate && \ + exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \ %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s" From a89c3af660516d7d51bf3e3aa71f5cad4f2675e0 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Fri, 28 Sep 2018 19:58:35 +0300 Subject: [PATCH 06/19] Fixed a typo inside docker-compose.yml --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6ebae20e7478..1f1f5de43d2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,8 +23,6 @@ services: cvat_redis: container_name: cvat_redis image: redis:4.0.5-alpine - container_name: cvat_db - image: postgres:10.3-alpine networks: default: aliases: From 39fa6c1f2e6c06371abe3f6134e2b2ee2eeac561 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Fri, 28 Sep 2018 20:15:33 +0300 Subject: [PATCH 07/19] Short-live container to configure kibana. --- analytics/docker-compose.yml | 9 +++++++++ analytics/kibana/setup.sh | 3 +++ 2 files changed, 12 insertions(+) create mode 100644 analytics/kibana/setup.sh diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index b65e67ba4464..5501ee384a05 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -26,8 +26,16 @@ services: ELK_VERSION: 6.4.0 ports: - "5601:5601" + depends_on: ['cvat_elasticsearch'] restart: always + cvat_kibana_setup: + image: centos:7 + container_name: cvat_kibana_setup + volumes: ['./analystics/kibana/setup.sh:/usr/local/bin/kibana-setup.sh:ro'] + depends_on: ['cvat_kibana'] + command: ['/bin/bash', '-c', '/usr/local/bin/kibana-setup.sh'] + cvat_logstash: container_name: cvat_logstash image: cvat_logstash @@ -41,6 +49,7 @@ services: ELK_VERSION: 6.4.0 http_proxy: ${http_proxy} https_proxy: ${https_proxy} + depends_on: ['cvat_elasticsearch'] restart: always cvat: diff --git a/analytics/kibana/setup.sh b/analytics/kibana/setup.sh new file mode 100644 index 000000000000..7a693aac0b81 --- /dev/null +++ b/analytics/kibana/setup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + + From ac8ac422287d0573b7a5ddadd659d0f921976f67 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Mon, 1 Oct 2018 19:09:16 +0300 Subject: [PATCH 08/19] Import kibana dashboards from export.json on startup. --- analytics/README.md | 0 analytics/docker-compose.yml | 12 ++++-- .../kibana/{config/cvat.json => export.json} | 0 analytics/kibana/setup.py | 40 +++++++++++++++++++ analytics/kibana/setup.sh | 3 -- 5 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 analytics/README.md rename analytics/kibana/{config/cvat.json => export.json} (100%) create mode 100644 analytics/kibana/setup.py delete mode 100644 analytics/kibana/setup.sh diff --git a/analytics/README.md b/analytics/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index 5501ee384a05..aa5ec07856f6 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -30,11 +30,15 @@ services: restart: always cvat_kibana_setup: - image: centos:7 container_name: cvat_kibana_setup - volumes: ['./analystics/kibana/setup.sh:/usr/local/bin/kibana-setup.sh:ro'] - depends_on: ['cvat_kibana'] - command: ['/bin/bash', '-c', '/usr/local/bin/kibana-setup.sh'] + image: cvat + volumes: ['./analytics/kibana:/home/django/kibana:ro'] + depends_on: ['cvat'] + working_dir: '/home/django' + entrypoint: ['bash', 'wait-for-it.sh', 'kibana:5601', '-t', '0', '--', + '/usr/bin/python3', 'kibana/setup.py', 'kibana/export.json'] + environment: + no_proxy: kibana,${no_proxy} cvat_logstash: container_name: cvat_logstash diff --git a/analytics/kibana/config/cvat.json b/analytics/kibana/export.json similarity index 100% rename from analytics/kibana/config/cvat.json rename to analytics/kibana/export.json diff --git a/analytics/kibana/setup.py b/analytics/kibana/setup.py new file mode 100644 index 000000000000..f1877ffec07c --- /dev/null +++ b/analytics/kibana/setup.py @@ -0,0 +1,40 @@ +#/usr/bin/env python + +import os +import argparse +import requests +import json + +def import_resources(host, port, cfg_file): + with open(cfg_file, 'r') as f: + for saved_object in json.load(f): + _id = saved_object["_id"] + _type = saved_object["_type"] + _doc = saved_object["_source"] + import_saved_object(host, port, _type, _id, _doc) + +def import_saved_object(host, port, _type, _id, data): + saved_objects_api = "http://{}:{}/api/saved_objects/{}/{}".format( + host, port, _type, _id) + request = requests.get(saved_objects_api) + if request.status_code == 404: + print("Creating {} as {}".format(_type, _id)) + request = requests.post(saved_objects_api, json={"attributes": data}, + headers={'kbn-xsrf': 'true'}) + else: + print("Updating {} named {}".format(_type, _id)) + request = requests.put(saved_objects_api, json={"attributes": data}, + headers={'kbn-xsrf': 'true'}) + request.raise_for_status() + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='import Kibana 6.x resources', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('export_file', metavar='FILE', + help='JSON export file with resources') + parser.add_argument('-p', '--port', metavar='PORT', default=5601, type=int, + help='port of Kibana instance') + parser.add_argument('-H', '--host', metavar='HOST', default='kibana', + help='host of Kibana instance') + args = parser.parse_args() + import_resources(args.host, args.port, args.export_file) diff --git a/analytics/kibana/setup.sh b/analytics/kibana/setup.sh deleted file mode 100644 index 7a693aac0b81..000000000000 --- a/analytics/kibana/setup.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - - From 6917a0a25fbdbb3ba235f0d0c075e4d85bcd7845 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Tue, 2 Oct 2018 15:11:44 +0300 Subject: [PATCH 09/19] Added logstash handler for cvat logger. --- cvat/apps/log_proxy/proxy_logger.py | 6 +++--- cvat/apps/log_proxy/views.py | 1 - cvat/requirements/base.txt | 1 + cvat/settings/base.py | 16 +++++++++++++++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/cvat/apps/log_proxy/proxy_logger.py b/cvat/apps/log_proxy/proxy_logger.py index d10398c568ff..5df6c924fc3b 100644 --- a/cvat/apps/log_proxy/proxy_logger.py +++ b/cvat/apps/log_proxy/proxy_logger.py @@ -38,7 +38,7 @@ def _get_task(self, tid): try: return Task.objects.get(pk=tid) except Exception: - raise Exception('Key must be task indentificator') + raise Exception('{} key must be a task identifier'.format(tid)) class ClientLogProxy(): class _HandlerType(Enum): @@ -67,7 +67,7 @@ def create_retry_session(retries=3, session=None, backoff_factor=0.3): parse_result = urlparse(log_server_url) if parse_result.scheme and 'http' not in parse_result.scheme: - raise Exception('unsuported annotation log destination') + raise Exception('unsupported annotation log destination') def http_log_handler(taskID, messages): r = create_retry_session().post(url=log_server_url, json=messages, verify=False) @@ -86,6 +86,6 @@ def _get_task_id(self, jid): job = Job.objects.select_related("segment__task").get(id=jid) return job.segment.task.id except: - raise Exception('Key must be job indentificator') + raise Exception('{} key must be a job identifier'.format(jid)) client_log_proxy = ClientLogProxy() diff --git a/cvat/apps/log_proxy/views.py b/cvat/apps/log_proxy/views.py index 645c20362499..245ad1414f05 100644 --- a/cvat/apps/log_proxy/views.py +++ b/cvat/apps/log_proxy/views.py @@ -11,7 +11,6 @@ import json -# Create your views here. @login_required() @permission_required('engine.view_task', raise_exception=True) def exception_receiver(request, jid): diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 8f6092361d2b..839d118833a3 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -22,3 +22,4 @@ scipy==1.0.1 sqlparse==0.2.4 django-sendfile==0.3.11 dj-pagination==2.3.2 +python-logstash==0.4.6 diff --git a/cvat/settings/base.py b/cvat/settings/base.py index cb937f473e22..ac8ff31a947a 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -189,7 +189,7 @@ }, 'file': { 'class': 'logging.handlers.RotatingFileHandler', - 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), + 'level': 'DEBUG', 'filename': os.path.join(BASE_DIR, 'logs', 'cvat.log'), 'formatter': 'standard', 'maxBytes': 1024*1024*50, # 50 MB @@ -204,6 +204,20 @@ }, } +log_server_url = os.getenv('DJANGO_LOG_SERVER_URL') +if log_server_url: + from urllib.parse import urlparse + url = urlparse(log_server_url) + LOGGING['handlers']['logstash'] = { + 'level': 'INFO', + 'class': 'logstash.TCPLogstashHandler', + 'host': url.hostname, + 'port': url.port, + 'version': 1, + 'message_type': 'django', + } + LOGGING['loggers']['cvat']['handlers'] += ['logstash'] + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ From b5c9b08e10f000315a2fd2f7f648bf0da749fb94 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Tue, 2 Oct 2018 18:23:18 +0300 Subject: [PATCH 10/19] Server logs (part 1) --- .vscode/launch.json | 5 ++++- analytics/logstash/logstash.conf | 11 ++--------- cvat/apps/engine/task.py | 2 +- cvat/apps/engine/views.py | 2 +- cvat/settings/base.py | 21 +++++++++++++++++---- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index e0ae80ab1858..73aa998b7174 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,6 +9,7 @@ "type": "python", "request": "launch", "stopOnEntry": false, + "debugStdLib": true, "pythonPath": "${config:python.pythonPath}", "program": "${workspaceRoot}/manage.py", "args": [ @@ -23,7 +24,7 @@ "DjangoDebugging" ], "cwd": "${workspaceFolder}", - "env": {}, + "env": {"DJANGO_LOG_SERVER_URL": "http://localhost:5000"}, "envFile": "${workspaceFolder}/.env", }, { @@ -44,6 +45,7 @@ "type": "python", "request": "launch", "stopOnEntry": false, + "debugStdLib": true, "pythonPath": "${config:python.pythonPath}", "program": "${workspaceRoot}/manage.py", "args": [ @@ -65,6 +67,7 @@ "name": "CVAT RQ - low", "type": "python", "request": "launch", + "debugStdLib": true, "stopOnEntry": false, "pythonPath": "${config:python.pythonPath}", "program": "${workspaceRoot}/manage.py", diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf index 16362184cafb..0aec9ce418f4 100644 --- a/analytics/logstash/logstash.conf +++ b/analytics/logstash/logstash.conf @@ -1,18 +1,11 @@ input { - http { + tcp { port => 5000 - response_headers => { - "Access-Control-Allow-Origin" => "*" - "Access-Control-Allow-Headers" => "Content-Type" - } + codec => json } } filter { - if [headers][request_method] == "OPTIONS" { - drop { } - } - prune { blacklist_names => ["headers", "host"] } diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index f6d755be5923..3800ea89a3a4 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -32,7 +32,7 @@ from . import models from .logging import task_logger, job_logger -global_logger = logging.getLogger(__name__) +global_logger = logging.getLogger('cvat') ############################# Low Level server API diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index f1edac086645..ad74edb4c09a 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -22,7 +22,7 @@ from requests.exceptions import RequestException from .logging import task_logger, job_logger -global_logger = logging.getLogger(__name__) +global_logger = logging.getLogger('cvat') ############################# High Level server API @login_required diff --git a/cvat/settings/base.py b/cvat/settings/base.py index ac8ff31a947a..7c2cf40afa35 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -187,18 +187,31 @@ 'class': 'logging.StreamHandler', 'formatter': 'standard', }, - 'file': { + 'server_file': { 'class': 'logging.handlers.RotatingFileHandler', 'level': 'DEBUG', - 'filename': os.path.join(BASE_DIR, 'logs', 'cvat.log'), + 'filename': os.path.join(BASE_DIR, 'logs', 'cvat_server.log'), + 'formatter': 'standard', + 'maxBytes': 1024*1024*50, # 50 MB + 'backupCount': 5, + }, + 'client_file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'level': 'DEBUG', + 'filename': os.path.join(BASE_DIR, 'logs', 'cvat_client.log'), 'formatter': 'standard', 'maxBytes': 1024*1024*50, # 50 MB 'backupCount': 5, } }, 'loggers': { - 'cvat': { - 'handlers': ['console', 'file'], + 'cvat.server': { + 'handlers': ['console', 'server_file'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), + }, + + 'cvat.client': { + 'handlers': ['client_file'], 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), } }, From f082084ae1c13e985cfb251d160c9a71318cf829 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Wed, 3 Oct 2018 19:22:21 +0300 Subject: [PATCH 11/19] Client and server logging is working throw python-logstash --- .vscode/launch.json | 5 +- README.md | 14 +--- analytics/docker-compose.yml | 3 +- cvat/apps/engine/logging.py | 93 ++++++++++++--------- cvat/apps/engine/static/engine/js/logger.js | 2 +- cvat/apps/engine/task.py | 4 +- cvat/apps/engine/urls.py | 1 + cvat/apps/engine/views.py | 18 ++-- cvat/apps/log_proxy/__init__.py | 5 -- cvat/apps/log_proxy/admin.py | 9 -- cvat/apps/log_proxy/apps.py | 11 --- cvat/apps/log_proxy/migrations/__init__.py | 5 -- cvat/apps/log_proxy/models.py | 9 -- cvat/apps/log_proxy/proxy_logger.py | 91 -------------------- cvat/apps/log_proxy/tests.py | 9 -- cvat/apps/log_proxy/urls.py | 12 --- cvat/apps/log_proxy/views.py | 24 ------ cvat/settings/base.py | 32 +++---- cvat/urls.py | 3 +- docker-compose.yml | 3 +- 20 files changed, 92 insertions(+), 261 deletions(-) delete mode 100644 cvat/apps/log_proxy/__init__.py delete mode 100644 cvat/apps/log_proxy/admin.py delete mode 100644 cvat/apps/log_proxy/apps.py delete mode 100644 cvat/apps/log_proxy/migrations/__init__.py delete mode 100644 cvat/apps/log_proxy/models.py delete mode 100644 cvat/apps/log_proxy/proxy_logger.py delete mode 100644 cvat/apps/log_proxy/tests.py delete mode 100644 cvat/apps/log_proxy/urls.py delete mode 100644 cvat/apps/log_proxy/views.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 73aa998b7174..bf157da7f12d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -24,7 +24,10 @@ "DjangoDebugging" ], "cwd": "${workspaceFolder}", - "env": {"DJANGO_LOG_SERVER_URL": "http://localhost:5000"}, + "env": { + "DJANGO_LOG_SERVER_HOST": "localhost", + "DJANGO_LOG_SERVER_PORT": 5000 + }, "envFile": "${workspaceFolder}/.env", }, { diff --git a/README.md b/README.md index 24fa1eab5dd8..0748f0b8a994 100644 --- a/README.md +++ b/README.md @@ -109,19 +109,13 @@ services: ``` ### Annotation logs -It is possible to proxy annotation logs from client to another server over http. For examlpe you can use Logstash. -To do that set DJANGO_LOG_SERVER_URL environment variable in cvat section of docker-compose.yml -file (or add this variable to docker-compose.override.yml). +It is possible to proxy annotation logs from client to ELK. To do that run the following command below: -```yml -version: "2.3" - -services: -cvat: - environment: - DJANGO_LOG_SERVER_URL: https://annotation.example.com:5000 +```bash +docker-compose -f docker-compose.yml -f analytics/docker-compose.yml up -d --build ``` + ### Share path You can use a share storage for data uploading during you are creating a task. To do that you can mount it to CVAT docker container. Example of docker-compose.override.yml for this purpose: diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index aa5ec07856f6..be34aa9982ca 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -58,5 +58,6 @@ services: cvat: environment: - DJANGO_LOG_SERVER_URL: "http://logstash:5000" + DJANGO_LOG_SERVER_HOST: "logstash" + DJANGO_LOG_SERVER_PORT: 5000 no_proxy: logstash,${no_proxy} diff --git a/cvat/apps/engine/logging.py b/cvat/apps/engine/logging.py index 8948821f529b..1cf774b8404e 100644 --- a/cvat/apps/engine/logging.py +++ b/cvat/apps/engine/logging.py @@ -1,19 +1,28 @@ - # Copyright (C) 2018 Intel Corporation # # SPDX-License-Identifier: MIT import os -import inspect import logging from . import models from cvat.settings.base import LOGGING +from cvat.apps.engine.models import Job, Task + +def _get_task(tid): + try: + return Task.objects.get(pk=tid) + except Exception: + raise Exception('{} key must be a task identifier'.format(tid)) +def _get_job(jid): + try: + return models.Job.objects.select_related("segment__task").get(id=jid) + except Exception: + raise Exception('{} key must be a job identifier'.format(jid)) class TaskLoggerStorage: def __init__(self): self._storage = dict() - self._formatter = logging.getLogger('task') def __getitem__(self, tid): if tid not in self._storage: @@ -21,33 +30,13 @@ def __getitem__(self, tid): return self._storage[tid] def _create_task_logger(self, tid): - task = self._get_task(tid) - if task is not None: - configuration = LOGGING.copy() - handler_configuration = configuration['handlers']['file'] - handler_configuration['filename'] = task.get_log_path() - configuration['handlers'] = { - 'file_{}'.format(tid): handler_configuration - } - configuration['loggers'] = { - 'task_{}'.format(tid): { - 'handlers': ['file_{}'.format(tid)], - 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), - } - } - - logging.config.dictConfig(configuration) - logger = logging.getLogger('task_{}'.format(tid)) - return logger - else: - raise Exception('Key must be task indentificator') - - def _get_task(self, tid): - try: - return models.Task.objects.get(pk=tid) - except Exception: - return None + task = _get_task(tid) + + logger = logging.getLogger('cvat.server.task_{}'.format(tid)) + server_file = logging.FileHandler(filename=task.get_log_path()) + logger.addHandler(server_file) + return logger class JobLoggerStorage: def __init__(self): @@ -59,17 +48,41 @@ def __getitem__(self, jid): return self._storage[jid] def _get_task_logger(self, jid): - job = self._get_job(jid) - if job is not None: - return task_logger[job.segment.task.id] - else: - raise Exception('Key must be job identificator') - - def _get_job(self, jid): - try: - return models.Job.objects.select_related("segment__task").get(id=jid) - except Exception: - return None + job = _get_job(jid) + return task_logger[job.segment.task.id] + +class TaskClientLoggerStorage: + def __init__(self): + self._storage = dict() + + def __getitem__(self, tid): + if tid not in self._storage: + self._storage[tid] = self._create_client_logger(tid) + return self._storage[tid] + + def _create_client_logger(self, tid): + task = _get_task(tid) + logger = logging.getLogger('cvat.client.task_{}'.format(tid)) + client_file = logging.FileHandler(filename=task.get_client_log_path()) + logger.addHandler(client_file) + + return logger + +class JobClientLoggerStorage: + def __init__(self): + self._storage = dict() + + def __getitem__(self, jid): + if jid not in self._storage: + self._storage[jid] = self._get_task_logger(jid) + return self._storage[jid] + + def _get_task_logger(self, jid): + job = _get_job(jid) + return task_client_logger[job.segment.task.id] task_logger = TaskLoggerStorage() job_logger = JobLoggerStorage() +global_logger = logging.getLogger('cvat.server') +job_client_logger = JobClientLoggerStorage() +task_client_logger = TaskClientLoggerStorage() \ No newline at end of file diff --git a/cvat/apps/engine/static/engine/js/logger.js b/cvat/apps/engine/static/engine/js/logger.js index 45bf95bfc181..28e7fecbd014 100644 --- a/cvat/apps/engine/static/engine/js/logger.js +++ b/cvat/apps/engine/static/engine/js/logger.js @@ -80,7 +80,7 @@ var LoggerHandler = function(applicationName, jobId) return new Promise( (resolve, reject) => { let xhr = new XMLHttpRequest(); - xhr.open('POST', '/logs/exception/' + this._jobId); + xhr.open('POST', '/save/exception/' + this._jobId); xhr.setRequestHeader('Content-Type', 'application/json'); xhr.setRequestHeader("X-CSRFToken", Cookies.get('csrftoken')); diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 3800ea89a3a4..782450bc4cba 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -30,9 +30,7 @@ from distutils.dir_util import copy_tree from . import models -from .logging import task_logger, job_logger - -global_logger = logging.getLogger('cvat') +from .logging import task_logger, job_logger, global_logger ############################# Low Level server API diff --git a/cvat/apps/engine/urls.py b/cvat/apps/engine/urls.py index cc1e3496cfa7..218f0411fb7f 100644 --- a/cvat/apps/engine/urls.py +++ b/cvat/apps/engine/urls.py @@ -22,4 +22,5 @@ path('save/annotation/task/', views.save_annotation_for_task), path('get/annotation/job/', views.get_annotation), path('get/username', views.get_username), + path('save/exception/', views.catch_client_exception) ] diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index ad74edb4c09a..7206e55776dc 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -5,7 +5,6 @@ import os import json -import logging import traceback from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse @@ -18,13 +17,19 @@ from . import annotation, task, models from cvat.settings.base import JS_3RDPARTY from cvat.apps.authentication.decorators import login_required -from cvat.apps.log_proxy.proxy_logger import client_log_proxy from requests.exceptions import RequestException -from .logging import task_logger, job_logger - -global_logger = logging.getLogger('cvat') +import logging +from .logging import task_logger, job_logger, global_logger, job_client_logger ############################# High Level server API +@login_required +@permission_required('engine.view_task', raise_exception=True) +def catch_client_exception(request, jid): + data = json.loads(request.body.decode('utf-8')) + for event in json.loads(data['exceptions']): + event.update({'name': 'cvat.client', 'level': logging.INFO}) + job_client_logger[jid].info("client", extra=event) + @login_required def dispatch_request(request): """An entry point to dispatch legacy requests""" @@ -243,7 +248,8 @@ def save_annotation_for_job(request, jid): if 'annotation' in data: annotation.save_job(jid, json.loads(data['annotation'])) if 'logs' in data: - client_log_proxy.push_logs(jid, json.loads(data['logs'])) + for event in json.loads(data['logs']): + job_client_logger[jid].info("client", extra=event) except RequestException as e: job_logger[jid].error("cannot send annotation logs for job {}".format(jid), exc_info=True) return HttpResponseBadRequest(str(e)) diff --git a/cvat/apps/log_proxy/__init__.py b/cvat/apps/log_proxy/__init__.py deleted file mode 100644 index d8e62e54b356..000000000000 --- a/cvat/apps/log_proxy/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - diff --git a/cvat/apps/log_proxy/admin.py b/cvat/apps/log_proxy/admin.py deleted file mode 100644 index af8dfc47525b..000000000000 --- a/cvat/apps/log_proxy/admin.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.contrib import admin - -# Register your models here. - diff --git a/cvat/apps/log_proxy/apps.py b/cvat/apps/log_proxy/apps.py deleted file mode 100644 index 6b456281f3b7..000000000000 --- a/cvat/apps/log_proxy/apps.py +++ /dev/null @@ -1,11 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.apps import AppConfig - - -class LogProxyConfig(AppConfig): - name = 'log_proxy' - diff --git a/cvat/apps/log_proxy/migrations/__init__.py b/cvat/apps/log_proxy/migrations/__init__.py deleted file mode 100644 index d8e62e54b356..000000000000 --- a/cvat/apps/log_proxy/migrations/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - diff --git a/cvat/apps/log_proxy/models.py b/cvat/apps/log_proxy/models.py deleted file mode 100644 index cdf3b0827bf1..000000000000 --- a/cvat/apps/log_proxy/models.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.db import models - -# Create your models here. - diff --git a/cvat/apps/log_proxy/proxy_logger.py b/cvat/apps/log_proxy/proxy_logger.py deleted file mode 100644 index 5df6c924fc3b..000000000000 --- a/cvat/apps/log_proxy/proxy_logger.py +++ /dev/null @@ -1,91 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.conf import settings -import os -import logging -import requests -import json -from urllib.parse import urlparse -from enum import Enum -from cvat.apps.engine.models import Job, Task - -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry - -class ClientLoggerStorage: - def __init__(self): - self._storage = dict() - self._formatter = logging.Formatter('%(message)s') - - def __getitem__(self, tid): - if tid not in self._storage: - self._storage[tid] = self._create_client_logger(tid) - return self._storage[tid] - - def _create_client_logger(self, tid): - task = self._get_task(tid) - logger = logging.getLogger(name='client_annotation_logger_{}'.format(tid)) - logger.setLevel(logging.INFO) - handler = logging.FileHandler(filename=task.get_client_log_path()) - handler.setFormatter(self._formatter) - logger.addHandler(handler) - return logger - - def _get_task(self, tid): - try: - return Task.objects.get(pk=tid) - except Exception: - raise Exception('{} key must be a task identifier'.format(tid)) - -class ClientLogProxy(): - class _HandlerType(Enum): - FILE = 1 - HTTP = 2 - - def __init__(self): - self._client_logger = ClientLoggerStorage() - def file_log_handler(tid, messages): - for event in messages: - self._client_logger[tid].info(json.dumps(event)) - - self._handlers = {self._HandlerType.FILE: file_log_handler} - - log_server_url = os.environ.get('DJANGO_LOG_SERVER_URL') - - def create_retry_session(retries=3, session=None, backoff_factor=0.3): - session = session or requests.Session() - retry = Retry(total=retries, backoff_factor=backoff_factor) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - return session - - if log_server_url: - parse_result = urlparse(log_server_url) - - if parse_result.scheme and 'http' not in parse_result.scheme: - raise Exception('unsupported annotation log destination') - - def http_log_handler(taskID, messages): - r = create_retry_session().post(url=log_server_url, json=messages, verify=False) - r.raise_for_status() - - self._handlers[self._HandlerType.HTTP] = http_log_handler - - def push_logs(self, jid, logs): - taskID = self._get_task_id(jid) - - for handler in self._handlers.values(): - handler(taskID, logs) - - def _get_task_id(self, jid): - try: - job = Job.objects.select_related("segment__task").get(id=jid) - return job.segment.task.id - except: - raise Exception('{} key must be a job identifier'.format(jid)) - -client_log_proxy = ClientLogProxy() diff --git a/cvat/apps/log_proxy/tests.py b/cvat/apps/log_proxy/tests.py deleted file mode 100644 index 53bc3b7adb85..000000000000 --- a/cvat/apps/log_proxy/tests.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.test import TestCase - -# Create your tests here. - diff --git a/cvat/apps/log_proxy/urls.py b/cvat/apps/log_proxy/urls.py deleted file mode 100644 index c483e7fdac66..000000000000 --- a/cvat/apps/log_proxy/urls.py +++ /dev/null @@ -1,12 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.urls import path -from . import views - -urlpatterns = [ - path('exception/', views.exception_receiver), -] - diff --git a/cvat/apps/log_proxy/views.py b/cvat/apps/log_proxy/views.py deleted file mode 100644 index 245ad1414f05..000000000000 --- a/cvat/apps/log_proxy/views.py +++ /dev/null @@ -1,24 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.http import HttpResponse, HttpResponseBadRequest -from django.contrib.auth.decorators import permission_required -from .proxy_logger import client_log_proxy -from cvat.apps.authentication.decorators import login_required - - -import json - -@login_required() -@permission_required('engine.view_task', raise_exception=True) -def exception_receiver(request, jid): - data = json.loads(request.body.decode('utf-8')) - try: - if 'exceptions' in data: - client_log_proxy.push_logs(jid, data['exceptions']) - except Exception as e: - return HttpResponseBadRequest(str(e)) - - return HttpResponse() diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 7c2cf40afa35..98ebc984e2d1 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -195,13 +195,13 @@ 'maxBytes': 1024*1024*50, # 50 MB 'backupCount': 5, }, - 'client_file': { - 'class': 'logging.handlers.RotatingFileHandler', - 'level': 'DEBUG', - 'filename': os.path.join(BASE_DIR, 'logs', 'cvat_client.log'), - 'formatter': 'standard', - 'maxBytes': 1024*1024*50, # 50 MB - 'backupCount': 5, + 'logstash': { + 'level': 'INFO', + 'class': 'logstash.TCPLogstashHandler', + 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), + 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 5000), + 'version': 1, + 'message_type': 'django', } }, 'loggers': { @@ -211,25 +211,15 @@ }, 'cvat.client': { - 'handlers': ['client_file'], + 'handlers': ['console'], 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), } }, } -log_server_url = os.getenv('DJANGO_LOG_SERVER_URL') -if log_server_url: - from urllib.parse import urlparse - url = urlparse(log_server_url) - LOGGING['handlers']['logstash'] = { - 'level': 'INFO', - 'class': 'logstash.TCPLogstashHandler', - 'host': url.hostname, - 'port': url.port, - 'version': 1, - 'message_type': 'django', - } - LOGGING['loggers']['cvat']['handlers'] += ['logstash'] +if os.getenv('DJANGO_LOG_SERVER_HOST'): + LOGGING['loggers']['cvat.server']['handlers'] += ['logstash'] + LOGGING['loggers']['cvat.client']['handlers'] += ['logstash'] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ diff --git a/cvat/urls.py b/cvat/urls.py index 657af2f45e4d..0d27ee7ab02f 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -30,8 +30,7 @@ path('dashboard/', include('cvat.apps.dashboard.urls')), path('django-rq/', include('django_rq.urls')), path('auth/', include('cvat.apps.authentication.urls')), - path('documentation/', include('cvat.apps.documentation.urls')), - path('logs/', include('cvat.apps.log_proxy.urls')) + path('documentation/', include('cvat.apps.documentation.urls')) ] if 'yes' == os.environ.get('TF_ANNOTATION', 'no'): diff --git a/docker-compose.yml b/docker-compose.yml index 1f1f5de43d2d..d55f64b4bd22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,8 @@ services: WITH_TESTS: "no" environment: DJANGO_MODWSGI_EXTRA_ARGS: "" - DJANGO_LOG_SERVER_URL: "" + DJANGO_LOG_SERVER_HOST: "" + DJANGO_LOG_SERVER_PORT: "" volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys From 17b1178732cbc2117cc66157734996c3d17a1bfb Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Wed, 3 Oct 2018 19:38:51 +0300 Subject: [PATCH 12/19] Save inside message json for events (cvat.client logger). --- analytics/logstash/logstash.conf | 10 ++++++++-- cvat/apps/engine/views.py | 5 ++--- cvat/settings/base.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf index 0aec9ce418f4..a17fca64f7a7 100644 --- a/analytics/logstash/logstash.conf +++ b/analytics/logstash/logstash.conf @@ -6,8 +6,14 @@ input { } filter { - prune { - blacklist_names => ["headers", "host"] + if [logger_name] =~ "cvat.client.*" { + json { + source => "message" + } + + prune { + blacklist_names => ["level", "host", "logger_name", "message", "path", "port", "stack_info", "tags"] + } } if [event] == "Send exception" { diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 7206e55776dc..5b5a2ba8d25c 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -27,8 +27,7 @@ def catch_client_exception(request, jid): data = json.loads(request.body.decode('utf-8')) for event in json.loads(data['exceptions']): - event.update({'name': 'cvat.client', 'level': logging.INFO}) - job_client_logger[jid].info("client", extra=event) + job_client_logger[jid].info(json.dumps(event)) @login_required def dispatch_request(request): @@ -249,7 +248,7 @@ def save_annotation_for_job(request, jid): annotation.save_job(jid, json.loads(data['annotation'])) if 'logs' in data: for event in json.loads(data['logs']): - job_client_logger[jid].info("client", extra=event) + job_client_logger[jid].info(json.dumps(event)) except RequestException as e: job_logger[jid].error("cannot send annotation logs for job {}".format(jid), exc_info=True) return HttpResponseBadRequest(str(e)) diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 98ebc984e2d1..1506625343ee 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -211,7 +211,7 @@ }, 'cvat.client': { - 'handlers': ['console'], + 'handlers': [], 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), } }, From a25fde7152e1bf40a12a92dd8f7ba89fef685c9e Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 4 Oct 2018 12:11:20 +0300 Subject: [PATCH 13/19] Filtered content of events, push client and server events into different indexes. --- analytics/logstash/logstash.conf | 83 +++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf index a17fca64f7a7..38103be444ab 100644 --- a/analytics/logstash/logstash.conf +++ b/analytics/logstash/logstash.conf @@ -6,39 +6,61 @@ input { } filter { - if [logger_name] =~ "cvat.client.*" { + if [logger_name] =~ /cvat.client/ { + # 1. Decode the event from json in 'message' field + # 2. Remove unnecessary field from it + # 3. Type it as client json { source => "message" } - prune { - blacklist_names => ["level", "host", "logger_name", "message", "path", "port", "stack_info", "tags"] - } - } + if [event] == "Send exception" { + aggregate { + task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}" + code => " + require 'time' - if [event] == "Send exception" { - aggregate { - task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}" - code => " - require 'time' + map['userid'] ||= event.get('userid'); + map['application'] ||= event.get('application'); + map['message'] ||= event.get('message'); + map['filename'] ||= event.get('filename'); + map['line'] ||= event.get('line'); + map['task'] ||= event.get('task'); - map['userid'] ||= event.get('userid'); - map['application'] ||= event.get('application'); - map['message'] ||= event.get('message'); - map['filename'] ||= event.get('filename'); - map['line'] ||= event.get('line'); - map['task'] ||= event.get('task'); + map['error_count'] ||= 0; + map['error_count'] += 1; - map['error_count'] ||= 0; - map['error_count'] += 1; + map['aggregated_message'] ||= ''; + time = Time.strptime(event.get('timestamp').to_s,'%Q').localtime('+03:00') + map['aggregated_message'] += time.to_s + '\n' + event.get('stack') + '\n\n\n';" - map['aggregated_message'] ||= ''; - time = Time.strptime(event.get('timestamp').to_s,'%Q').localtime('+03:00') - map['aggregated_message'] += time.to_s + '\n' + event.get('stack') + '\n\n\n';" + timeout => 3600 + timeout_tags => ['send_email_notification'] + push_map_as_event_on_timeout => true + } + } else { + prune { + blacklist_names => ["stack_info"] + } + } + + prune { + blacklist_names => ["level", "host", "logger_name", "message", "path", + "port"] + } - timeout => 3600 - timeout_tags => ['send_email_notification'] - push_map_as_event_on_timeout => true + mutate { + replace => { "type" => "client" } + } + } else if [logger_name] =~ /cvat.server/ { + # 1. Remove unnecessary field from it + # 2. Type it as server + prune { + blacklist_names => ["host", "port"] + } + + mutate { + replace => { "type" => "server" } } } @@ -52,8 +74,15 @@ output { codec => rubydebug } - elasticsearch { - hosts => ["elasticsearch:9200"] - index => "cvat-client" + if [type] == "client" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "cvat.client" + } + } else if [type] == "server" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "cvat.server" + } } } From 7e5e6c43cda1677d32952def54ed23c6b9994819 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 4 Oct 2018 12:45:19 +0300 Subject: [PATCH 14/19] Fixed a typo and removed unnecessary files. --- analytics/logstash/logstash.conf | 14 +++++--------- cvat/apps/engine/views.py | 6 ++++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf index 38103be444ab..cec1115a9202 100644 --- a/analytics/logstash/logstash.conf +++ b/analytics/logstash/logstash.conf @@ -14,6 +14,10 @@ filter { source => "message" } + date { + match => ["timestamp", "UNIX", "UNIX_MS"] + } + if [event] == "Send exception" { aggregate { task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}" @@ -38,15 +42,11 @@ filter { timeout_tags => ['send_email_notification'] push_map_as_event_on_timeout => true } - } else { - prune { - blacklist_names => ["stack_info"] - } } prune { blacklist_names => ["level", "host", "logger_name", "message", "path", - "port"] + "port", "stack_info", "timestamp"] } mutate { @@ -63,10 +63,6 @@ filter { replace => { "type" => "server" } } } - - date { - match => ["timestamp", "UNIX", "UNIX_MS"] - } } output { diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 5b5a2ba8d25c..3caae2809683 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -26,8 +26,10 @@ @permission_required('engine.view_task', raise_exception=True) def catch_client_exception(request, jid): data = json.loads(request.body.decode('utf-8')) - for event in json.loads(data['exceptions']): - job_client_logger[jid].info(json.dumps(event)) + for event in data['exceptions']: + job_client_logger[jid].error(json.dumps(event)) + + return HttpResponse() @login_required def dispatch_request(request): From 822e10c0c5a1f60e6b30194f3c41144a486ac331 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 4 Oct 2018 14:04:06 +0300 Subject: [PATCH 15/19] Minor polishing of events and dashboards. --- analytics/docker-compose.yml | 5 +- analytics/kibana/export.json | 193 ++++++++++++++++++------------- analytics/logstash/logstash.conf | 3 +- 3 files changed, 118 insertions(+), 83 deletions(-) diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index be34aa9982ca..9e279c99d807 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -35,10 +35,11 @@ services: volumes: ['./analytics/kibana:/home/django/kibana:ro'] depends_on: ['cvat'] working_dir: '/home/django' - entrypoint: ['bash', 'wait-for-it.sh', 'kibana:5601', '-t', '0', '--', + entrypoint: ['bash', 'wait-for-it.sh', 'elasticsearch:9200', '-t', '0', '--', + '/bin/bash', 'wait-for-it.sh', 'kibana:5601', '-t', '0', '--', '/usr/bin/python3', 'kibana/setup.py', 'kibana/export.json'] environment: - no_proxy: kibana,${no_proxy} + no_proxy: elasticsearch,kibana,${no_proxy} cvat_logstash: container_name: cvat_logstash diff --git a/analytics/kibana/export.json b/analytics/kibana/export.json index ccfb29d743a3..7b5f80c78ef9 100644 --- a/analytics/kibana/export.json +++ b/analytics/kibana/export.json @@ -1,148 +1,181 @@ [ { - "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8", - "_type": "dashboard", + "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8", + "_type": "visualization", "_source": { - "title": "Monitoring", - "hits": 0, "description": "", - "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", - "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", - "version": 1, - "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - } + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + }, + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"User\",\"field\":\"userid.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"3-bucket\",\"params\":{\"customInterval\":\"2h\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"date_histogram\"},\"customLabel\":\"Activity\",\"customMetric\":{\"enabled\":true,\"id\":\"3-metric\",\"params\":{},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Working Time (h)\",\"field\":\"working time\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Users\",\"type\":\"table\"}", + "title": "Users", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8", - "_type": "dashboard", + "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8", + "_type": "visualization", "_source": { - "title": "Managment", - "hits": 0, "description": "", - "panelsJSON": "[{\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":33,\"i\":\"1\"},\"version\":\"6.4.0\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":33,\"i\":\"2\"},\"version\":\"6.4.0\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":20,\"i\":\"3\"},\"version\":\"6.4.0\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}}]", - "optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}", - "version": 1, - "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - } + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}" + }, + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"4\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"min\"},{\"enabled\":true,\"id\":\"5\",\"params\":{\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"max\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events duration\",\"type\":\"table\"}", + "title": "Events duration", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8", - "_type": "index-pattern", + "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8", + "_type": "visualization", "_source": { - "title": "cvat*", - "timeFieldName": "@timestamp", - "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", - "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}" + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "visState": "{\"title\":\"Working Day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}", + "title": "Working Day", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8", - "_type": "search", + "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", "_source": { - "title": "Exceptions table", + "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", + "timeRestore": false, "description": "", "hits": 0, - "columns": [ - "event" - ], - "sort": [ - "@timestamp", - "desc" - ], - "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}" - } + "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + }, + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", + "title": "Monitoring", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8", - "_type": "visualization", + "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", "_source": { - "title": "Exceptions timeline", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"1-bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"filters\"},\"customLabel\":\"Exceptions\",\"customMetric\":{\"enabled\":true,\"id\":\"1-metric\",\"params\":{\"customLabel\":\"Exceptions\"},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Time\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"title\":\"Exceptions timeline\",\"type\":\"histogram\"}", - "uiStateJSON": "{}", + "optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}", + "timeRestore": false, "description": "", - "version": 1, + "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - } + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "panelsJSON": "[{\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":33,\"i\":\"1\"},\"version\":\"6.4.0\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":33,\"i\":\"2\"},\"version\":\"6.4.0\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":20,\"i\":\"3\"},\"version\":\"6.4.0\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}}]", + "title": "Managment", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8", + "_id": "65918380-c244-11e8-8e1b-758ef07f6de8", "_type": "visualization", "_source": { - "title": "Users", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"User\",\"field\":\"userid.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"3-bucket\",\"params\":{\"customInterval\":\"2h\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"date_histogram\"},\"customLabel\":\"Activity\",\"customMetric\":{\"enabled\":true,\"id\":\"3-metric\",\"params\":{},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Working Time (h)\",\"field\":\"working time\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Users\",\"type\":\"table\"}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", - "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - } + }, + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events count\",\"type\":\"table\"}", + "title": "Events count", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8", + "_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8", "_type": "visualization", "_source": { - "title": "Events duration", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"4\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"min\"},{\"enabled\":true,\"id\":\"5\",\"params\":{\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"max\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events duration\",\"type\":\"table\"}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", - "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}" - } + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "uiStateJSON": "{}", + "visState": "{\"title\":\"Users Activity\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}", + "title": "Users Activity", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8", + "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8", "_type": "visualization", "_source": { - "title": "Working Day", - "visState": "{\"title\":\"Working Day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "title": "Timeline for exceptions", + "visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucketAgg\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}]}", + "uiStateJSON": "{}", "description": "", "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" } + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "65918380-c244-11e8-8e1b-758ef07f6de8", - "_type": "visualization", + "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8", + "_type": "index-pattern", "_source": { - "title": "Events count", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events count\",\"type\":\"table\"}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - } + "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}", + "title": "cvat*", + "timeFieldName": "@timestamp" + }, + "_meta": { + "savedObjectVersion": 2 } }, { - "_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8", - "_type": "visualization", + "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8", + "_type": "search", "_source": { - "title": "Users Activity", - "visState": "{\"title\":\"Users Activity\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}", - "uiStateJSON": "{}", + "title": "Table with exceptions", "description": "", + "hits": 0, + "columns": [ + "task", + "type", + "userid", + "stack" + ], + "sort": [ + "@timestamp", + "desc" + ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}" } + }, + "_meta": { + "savedObjectVersion": 2 } } ] \ No newline at end of file diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf index cec1115a9202..f0ecb4caf271 100644 --- a/analytics/logstash/logstash.conf +++ b/analytics/logstash/logstash.conf @@ -16,6 +16,7 @@ filter { date { match => ["timestamp", "UNIX", "UNIX_MS"] + remove_field => "timestamp" } if [event] == "Send exception" { @@ -46,7 +47,7 @@ filter { prune { blacklist_names => ["level", "host", "logger_name", "message", "path", - "port", "stack_info", "timestamp"] + "port", "stack_info"] } mutate { From 3f4706b190c43d25f5cc8038f25111eb2050b179 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 4 Oct 2018 14:05:02 +0300 Subject: [PATCH 16/19] Removed debugging code from .vscode/launch.json --- .vscode/launch.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index bf157da7f12d..ac399cc33425 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -24,10 +24,6 @@ "DjangoDebugging" ], "cwd": "${workspaceFolder}", - "env": { - "DJANGO_LOG_SERVER_HOST": "localhost", - "DJANGO_LOG_SERVER_PORT": 5000 - }, "envFile": "${workspaceFolder}/.env", }, { From ba66ac2935330dff771d18888976372d2795e8c2 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 4 Oct 2018 16:31:16 +0300 Subject: [PATCH 17/19] Default is discover, new visualizations. --- analytics/kibana/export.json | 171 +++++++++++++++++++---------------- analytics/kibana/kibana.yml | 1 + 2 files changed, 95 insertions(+), 77 deletions(-) diff --git a/analytics/kibana/export.json b/analytics/kibana/export.json index 7b5f80c78ef9..65b1b8702461 100644 --- a/analytics/kibana/export.json +++ b/analytics/kibana/export.json @@ -1,32 +1,34 @@ [ { - "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8", + "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8", "_type": "visualization", "_source": { + "title": "Timeline for exceptions", + "visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucketAgg\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}]}", + "uiStateJSON": "{}", "description": "", + "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - }, - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"User\",\"field\":\"userid.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customBucket\":{\"enabled\":true,\"id\":\"3-bucket\",\"params\":{\"customInterval\":\"2h\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":{\"aggFilter\":[],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"bucketAgg\",\"params\":[],\"title\":\"Bucket Agg\"},\"type\":\"date_histogram\"},\"customLabel\":\"Activity\",\"customMetric\":{\"enabled\":true,\"id\":\"3-metric\",\"params\":{},\"schema\":{\"aggFilter\":[\"!top_hits\",\"!percentiles\",\"!percentile_ranks\",\"!median\",\"!std_dev\",\"!sum_bucket\",\"!avg_bucket\",\"!min_bucket\",\"!max_bucket\",\"!derivative\",\"!moving_avg\",\"!serial_diff\",\"!cumulative_sum\"],\"deprecate\":false,\"editor\":false,\"group\":\"none\",\"max\":null,\"min\":0,\"name\":\"metricAgg\",\"params\":[],\"title\":\"Metric Agg\"},\"type\":\"count\"}},\"schema\":\"metric\",\"type\":\"sum_bucket\"},{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Working Time (h)\",\"field\":\"working time\"},\"schema\":\"metric\",\"type\":\"sum\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Users\",\"type\":\"table\"}", - "title": "Users", - "version": 1 + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } }, "_meta": { "savedObjectVersion": 2 } }, { - "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8", - "_type": "visualization", + "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", "_source": { + "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", + "timeRestore": false, "description": "", + "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}" + "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" }, - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"4\",\"params\":{\"customLabel\":\"\",\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"min\"},{\"enabled\":true,\"id\":\"5\",\"params\":{\"field\":\"duration\"},\"schema\":\"metric\",\"type\":\"max\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events duration\",\"type\":\"table\"}", - "title": "Events duration", + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", + "title": "Monitoring", "version": 1 }, "_meta": { @@ -34,55 +36,47 @@ } }, { - "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8", + "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8", "_type": "visualization", "_source": { + "title": "List of users", + "visState": "{\"title\":\"List of users\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"3-bucket\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"bucketAgg\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},\"customMetric\":{\"id\":\"3-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{}},\"customLabel\":\"Activity\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"working time\",\"customLabel\":\"Working Time (h)\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", + "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "visState": "{\"title\":\"Working Day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}", - "title": "Working Day", - "version": 1 + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } }, "_meta": { "savedObjectVersion": 2 } }, { - "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8", - "_type": "dashboard", + "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8", + "_type": "visualization", "_source": { - "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", - "timeRestore": false, + "title": "Duration of events", + "visState": "{\"title\":\"Duration of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", - "hits": 0, + "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - }, - "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", - "title": "Monitoring", - "version": 1 + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}" + } }, "_meta": { "savedObjectVersion": 2 } }, { - "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8", - "_type": "dashboard", + "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8", + "_type": "index-pattern", "_source": { - "optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}", - "timeRestore": false, - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "panelsJSON": "[{\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":33,\"i\":\"1\"},\"version\":\"6.4.0\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":33,\"i\":\"2\"},\"version\":\"6.4.0\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":20,\"i\":\"3\"},\"version\":\"6.4.0\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"embeddableConfig\":{}}]", - "title": "Managment", - "version": 1 + "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}", + "title": "cvat*", + "timeFieldName": "@timestamp" }, "_meta": { "savedObjectVersion": 2 @@ -92,14 +86,14 @@ "_id": "65918380-c244-11e8-8e1b-758ef07f6de8", "_type": "visualization", "_source": { + "title": "Number of events", + "visState": "{\"title\":\"Number of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", + "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" - }, - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", - "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Action\",\"field\":\"event.keyword\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":100},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Events count\",\"type\":\"table\"}", - "title": "Events count", - "version": 1 + } }, "_meta": { "savedObjectVersion": 2 @@ -109,26 +103,52 @@ "_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8", "_type": "visualization", "_source": { + "title": "Activity of users", + "visState": "{\"title\":\"Activity of users\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}", + "uiStateJSON": "{}", "description": "", + "version": 1, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "uiStateJSON": "{}", - "visState": "{\"title\":\"Users Activity\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}", - "title": "Users Activity", - "version": 1 + } }, "_meta": { "savedObjectVersion": 2 } }, { - "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8", + "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8", + "_type": "search", + "_source": { + "title": "Table with exceptions", + "description": "", + "hits": 0, + "columns": [ + "task", + "type", + "userid", + "stack" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "b6339c70-c7d4-11e8-a035-258d2bd7d91f", "_type": "visualization", "_source": { - "title": "Timeline for exceptions", - "visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucketAgg\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}]}", - "uiStateJSON": "{}", + "title": "Working calendar", + "visState": "{\"title\":\"Working calendar\",\"type\":\"heatmap\",\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Users\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}}]}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 8\":\"rgb(165,0,38)\",\"8 - 15\":\"rgb(249,142,82)\",\"15 - 23\":\"rgb(255,255,190)\",\"23 - 30\":\"rgb(135,203,103)\"}}}", "description": "", "version": 1, "kibanaSavedObjectMeta": { @@ -140,38 +160,35 @@ } }, { - "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8", - "_type": "index-pattern", + "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", "_source": { - "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", - "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}", - "title": "cvat*", - "timeFieldName": "@timestamp" + "title": "Managment", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"1\"},\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"2\"},\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":21,\"w\":24,\"h\":16,\"i\":\"3\"},\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"gridData\":{\"x\":0,\"y\":21,\"w\":24,\"h\":16,\"i\":\"4\"},\"version\":\"6.4.0\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"id\":\"b6339c70-c7d4-11e8-a035-258d2bd7d91f\",\"embeddableConfig\":{}}]", + "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } }, "_meta": { "savedObjectVersion": 2 } }, { - "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8", - "_type": "search", + "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8", + "_type": "visualization", "_source": { - "title": "Table with exceptions", + "title": "Working day", + "visState": "{\"title\":\"Working day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "description": "", - "hits": 0, - "columns": [ - "task", - "type", - "userid", - "stack" - ], - "sort": [ - "@timestamp", - "desc" - ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}" + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" } }, "_meta": { diff --git a/analytics/kibana/kibana.yml b/analytics/kibana/kibana.yml index 30f881e93000..29f931b4230b 100644 --- a/analytics/kibana/kibana.yml +++ b/analytics/kibana/kibana.yml @@ -1,3 +1,4 @@ server.host: 0.0.0.0 elasticsearch.url: http://elasticsearch:9200 elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ] +kibana.defaultAppId: "discover" From 9850f0066037a9376651518638bcfda4791a5363 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 4 Oct 2018 18:48:21 +0300 Subject: [PATCH 18/19] Make comments more readable inside logger.js. --- cvat/apps/engine/static/engine/js/logger.js | 108 +++++++++++++------- 1 file changed, 69 insertions(+), 39 deletions(-) diff --git a/cvat/apps/engine/static/engine/js/logger.js b/cvat/apps/engine/static/engine/js/logger.js index 28e7fecbd014..e26aef9de768 100644 --- a/cvat/apps/engine/static/engine/js/logger.js +++ b/cvat/apps/engine/static/engine/js/logger.js @@ -202,25 +202,31 @@ var LoggerHandler = function(applicationName, jobId) /* -Log message has simple json format - each message is set of "key" : "value" pairs inside curly braces - {"key1" : "string_value", "key2" : number_value, ...} -Value may be string or number (see json spec) -required fields for all event types: +Log message has simple json format - each message is set of "key" : "value" +pairs inside curly braces - {"key1" : "string_value", "key2" : number_value, +...} Value may be string or number (see json spec) required fields for all event +types: NAME TYPE DESCRIPTION "event" string see EventType enum description of possible values. -"timestamp" number timestamp in UNIX format - the number of seconds or milliseconds that have elapsed since 00:00:00 Thursday, 1 January 1970 +"timestamp" number timestamp in UNIX format - the number of seconds + or milliseconds that have elapsed since 00:00:00 + Thursday, 1 January 1970 "application" string application name "userid" string Unique userid "task" string Unique task id. (Is expected corresponding Jira task id) -"count" is requiered field for "Add object", "Delete object", "Copy track", "Propagate object", "Merge objecrs", "Undo action" and "Redo action" -events with number value. +"count" is requiered field for "Add object", "Delete object", "Copy track", +"Propagate object", "Merge objecrs", "Undo action" and "Redo action" events with +number value. -Example : { "event" : "Add object", "timestamp" : 1486040342867, "application" : "CVAT", "duration" : 4200, "userid" : "ESAZON1X-MOBL", "count" : 1, "type" : "bounding box" } +Example : { "event" : "Add object", "timestamp" : 1486040342867, "application" : +"CVAT", "duration" : 4200, "userid" : "ESAZON1X-MOBL", "count" : 1, "type" : +"bounding box" } -Types of supported events. -Minimum subset of events to generate simple report are Logger.EventType.addObject, Logger.EventType.deleteObject and Logger.EventType.sendTaskInfo. -Value of "count" property should be a number. +Types of supported events. Minimum subset of events to generate simple report +are Logger.EventType.addObject, Logger.EventType.deleteObject and +Logger.EventType.sendTaskInfo. Value of "count" property should be a number. */ var Logger = { @@ -276,50 +282,67 @@ var Logger = { EventType: { // dumped as "Paste object". There are no additional required fields. pasteObject: 0, - // dumped as "Change attribute". There are no additional required fields. + // dumped as "Change attribute". There are no additional required + // fields. changeAttribute: 1, // dumped as "Drag object". There are no additional required fields. dragObject: 2, - // dumped as "Delete object". "count" is required field, value of deleted objects should be positive number. + // dumped as "Delete object". "count" is required field, value of + // deleted objects should be positive number. deleteObject: 3, // dumped as "Press shortcut". There are no additional required fields. pressShortcut: 4, // dumped as "Resize object". There are no additional required fields. resizeObject: 5, - // dumped as "Send logs". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Send logs". It's expected that event has "duration" field, + // but it isn't necessary. sendLogs: 6, - // dumped as "Save job". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Save job". It's expected that event has "duration" field, + // but it isn't necessary. saveJob: 7, // dumped as "Jump frame". There are no additional required fields. jumpFrame: 8, - // dumped as "Draw object". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Draw object". It's expected that event has "duration" + // field, but it isn't necessary. drawObject: 9, // dumped as "Change label". changeLabel: 10, - // dumped as "Send task info". "track count", "frame count", "object count" are required fields. It's expected that event has "current_frame" field. + // dumped as "Send task info". "track count", "frame count", "object + // count" are required fields. It's expected that event has + // "current_frame" field. sendTaskInfo: 11, - // dumped as "Load job". "track count", "frame count", "object count" are required fields. It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Load job". "track count", "frame count", "object count" + // are required fields. It's expected that event has "duration" field, + // but it isn't necessary. loadJob: 12, - // dumped as "Move image". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Move image". It's expected that event has "duration" + // field, but it isn't necessary. moveImage: 13, - // dumped as "Zoom image". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Zoom image". It's expected that event has "duration" + // field, but it isn't necessary. zoomImage: 14, // dumped as "Lock object". There are no additional required fields. lockObject: 15, - // dumped as "Merge objects". "count" is required field with positive or negative number value. + // dumped as "Merge objects". "count" is required field with positive or + // negative number value. mergeObjects: 16, // dumped as "Copy object". "count" is required field with number value. copyObject: 17, - // dumped as "Propagate object". "count" is required field with number value. + // dumped as "Propagate object". "count" is required field with number + // value. propagateObject: 18, - // dumped as "Undo action". "count" is required field with positive or negative number value. + // dumped as "Undo action". "count" is required field with positive or + // negative number value. undoAction: 19, - // dumped as "Redo action". "count" is required field with positive or negative number value. + // dumped as "Redo action". "count" is required field with positive or + // negative number value. redoAction: 20, - // dumped as "Send user activity". "working_time" is required field with positive number value. + // dumped as "Send user activity". "working_time" is required field with + // positive number value. sendUserActivity: 21, - // dumped as "Send exception". Use to send any exception events to the server. - // "message", "filename", "line" are mandatory fields. "stack" and "column" are optional. + // dumped as "Send exception". Use to send any exception events to the + // server. "message", "filename", "line" are mandatory fields. "stack" + // and "column" are optional. sendException: 22, // dumped as "Change frame". There are no additional required fields. changeFrame: 23, @@ -356,10 +379,12 @@ var Logger = { /** * Logger.addContinuedEvent Use to add log event with duration field. - * Duration will be calculated automatically when LogEvent.close() method of returned Object will be called. - * Note: in case of LogEvent.close() method will not be callsed event will not be sended to server + * Duration will be calculated automatically when LogEvent.close() method of + * returned Object will be called. Note: in case of LogEvent.close() method + * will not be callsed event will not be sent to server * @param {Logger.EventType} type Event Type - * @param {Object} values Any event values, for example {count: 1, label: 'vehicle'} + * @param {Object} values Any event values, for example {count: 1, label: + * 'vehicle'} * @return {LogEvent} instance of LogEvent * @static */ @@ -370,7 +395,8 @@ var Logger = { /** * Logger.shortkeyLogDecorator use for decorating the shortkey handlers. - * This decorator just create appropriate log event and close it when decored function will performed. + * This decorator just create appropriate log event and close it when + * decored function will performed. * @param {Function} decoredFunc is function for decorating * @return {Function} is decorated decoredFunc * @static @@ -387,7 +413,7 @@ var Logger = { }, /** - * Logger.sendLogs Try to send exception logs to the server immediatly. + * Logger.sendLogs Try to send exception logs to the server immediately. * @return {Promise} * @param {LogEvent} exceptionEvent * @static @@ -414,7 +440,8 @@ var Logger = { }, /** - * Logger.setUsername just set username property which will be added to all log messages + * Logger.setUsername just set username property which will be added to all + * log messages * @param {String} username * @static */ @@ -423,7 +450,8 @@ var Logger = { this._logger.setUsername(username); }, - /** Logger.updateUserActivityTimer method updates internal timer for working time calculation logic + /** Logger.updateUserActivityTimer method updates internal timer for working + * time calculation logic * @static */ updateUserActivityTimer: function() @@ -431,11 +459,12 @@ var Logger = { this._logger.updateTimer(); }, - /** Logger.setTimeThreshold set time threshold in ms for EventType. - * If time interval betwwen incoming log events less than threshold events will be collapsed. - * Note that result event will have timestamp of first event, - * In case of time threshold used for continued event duration will be difference between - * first and last event timestamps and other fields from last event. + /** Logger.setTimeThreshold set time threshold in ms for EventType. If time + * interval betwwen incoming log events less than threshold events will be + * collapsed. Note that result event will have timestamp of first event, In + * case of time threshold used for continued event duration will be + * difference between first and last event timestamps and other fields from + * last event. * @static * @param {Logger.EventType} eventType * @param {Number} threshold @@ -445,7 +474,8 @@ var Logger = { this._logger.setTimeThreshold(eventType, threshold); }, - /** Logger._eventTypeToString private method to transform Logger.EventType to string + /** Logger._eventTypeToString private method to transform Logger.EventType + * to string * @param {Logger.EventType} type Event Type * @return {String} string reppresentation of Logger.EventType * @static From fd463917f3107c8a2146ee0f98676a93e0c65755 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Mon, 8 Oct 2018 18:18:51 +0300 Subject: [PATCH 19/19] Added a path for backups. --- analytics/elasticsearch/elasticsearch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/analytics/elasticsearch/elasticsearch.yml b/analytics/elasticsearch/elasticsearch.yml index c7c90a58bab7..a95860891f51 100644 --- a/analytics/elasticsearch/elasticsearch.yml +++ b/analytics/elasticsearch/elasticsearch.yml @@ -1,2 +1,3 @@ http.host: 0.0.0.0 script.painless.regex.enabled: true +path.repo: ["/usr/share/elasticsearch/backup"]