diff --git a/.vscode/launch.json b/.vscode/launch.json index e0ae80ab1858..ac399cc33425 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,6 +9,7 @@ "type": "python", "request": "launch", "stopOnEntry": false, + "debugStdLib": true, "pythonPath": "${config:python.pythonPath}", "program": "${workspaceRoot}/manage.py", "args": [ @@ -23,7 +24,6 @@ "DjangoDebugging" ], "cwd": "${workspaceFolder}", - "env": {}, "envFile": "${workspaceFolder}/.env", }, { @@ -44,6 +44,7 @@ "type": "python", "request": "launch", "stopOnEntry": false, + "debugStdLib": true, "pythonPath": "${config:python.pythonPath}", "program": "${workspaceRoot}/manage.py", "args": [ @@ -65,6 +66,7 @@ "name": "CVAT RQ - low", "type": "python", "request": "launch", + "debugStdLib": true, "stopOnEntry": false, "pythonPath": "${config:python.pythonPath}", "program": "${workspaceRoot}/manage.py", diff --git a/README.md b/README.md index 24fa1eab5dd8..0748f0b8a994 100644 --- a/README.md +++ b/README.md @@ -109,19 +109,13 @@ services: ``` ### Annotation logs -It is possible to proxy annotation logs from client to another server over http. For examlpe you can use Logstash. -To do that set DJANGO_LOG_SERVER_URL environment variable in cvat section of docker-compose.yml -file (or add this variable to docker-compose.override.yml). +It is possible to proxy annotation logs from client to ELK. To do that run the following command below: -```yml -version: "2.3" - -services: -cvat: - environment: - DJANGO_LOG_SERVER_URL: https://annotation.example.com:5000 +```bash +docker-compose -f docker-compose.yml -f analytics/docker-compose.yml up -d --build ``` + ### Share path You can use a share storage for data uploading during you are creating a task. To do that you can mount it to CVAT docker container. Example of docker-compose.override.yml for this purpose: diff --git a/analytics/README.md b/analytics/README.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml new file mode 100644 index 000000000000..9e279c99d807 --- /dev/null +++ b/analytics/docker-compose.yml @@ -0,0 +1,64 @@ +version: '2.3' +services: + cvat_elasticsearch: + container_name: cvat_elasticsearch + image: cvat_elasticsearch + networks: + default: + aliases: + - elasticsearch + build: + context: ./analytics/elasticsearch + args: + ELK_VERSION: 6.4.0 + restart: always + + cvat_kibana: + container_name: cvat_kibana + image: cvat_kibana + networks: + default: + aliases: + - kibana + build: + context: ./analytics/kibana + args: + ELK_VERSION: 6.4.0 + ports: + - "5601:5601" + depends_on: ['cvat_elasticsearch'] + restart: always + + cvat_kibana_setup: + container_name: cvat_kibana_setup + image: cvat + volumes: ['./analytics/kibana:/home/django/kibana:ro'] + depends_on: ['cvat'] + working_dir: '/home/django' + entrypoint: ['bash', 'wait-for-it.sh', 'elasticsearch:9200', '-t', '0', '--', + '/bin/bash', 'wait-for-it.sh', 'kibana:5601', '-t', '0', '--', + '/usr/bin/python3', 'kibana/setup.py', 'kibana/export.json'] + environment: + no_proxy: elasticsearch,kibana,${no_proxy} + + cvat_logstash: + container_name: cvat_logstash + image: cvat_logstash + networks: + default: + aliases: + - logstash + build: + context: ./analytics/logstash + args: + ELK_VERSION: 6.4.0 + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + depends_on: ['cvat_elasticsearch'] + restart: always + + cvat: + environment: + DJANGO_LOG_SERVER_HOST: "logstash" + DJANGO_LOG_SERVER_PORT: 5000 + no_proxy: logstash,${no_proxy} diff --git a/analytics/elasticsearch/Dockerfile b/analytics/elasticsearch/Dockerfile new file mode 100644 index 000000000000..e015784449af --- /dev/null +++ b/analytics/elasticsearch/Dockerfile @@ -0,0 +1,3 @@ +ARG ELK_VERSION +FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION} +COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/ diff --git a/analytics/elasticsearch/elasticsearch.yml b/analytics/elasticsearch/elasticsearch.yml new file mode 100644 index 000000000000..a95860891f51 --- /dev/null +++ b/analytics/elasticsearch/elasticsearch.yml @@ -0,0 +1,3 @@ +http.host: 0.0.0.0 +script.painless.regex.enabled: true +path.repo: ["/usr/share/elasticsearch/backup"] diff --git a/analytics/kibana/Dockerfile b/analytics/kibana/Dockerfile new file mode 100644 index 000000000000..f7982dffc29d --- /dev/null +++ b/analytics/kibana/Dockerfile @@ -0,0 +1,5 @@ +ARG ELK_VERSION +FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION} +COPY kibana.yml /usr/share/kibana/config/ + + diff --git a/analytics/kibana/export.json b/analytics/kibana/export.json new file mode 100644 index 000000000000..65b1b8702461 --- /dev/null +++ b/analytics/kibana/export.json @@ -0,0 +1,198 @@ +[ + { + "_id": "3ade53d0-c23e-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Timeline for exceptions", + "visState": "{\"title\":\"Timeline for exceptions\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Exceptions\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Exceptions\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"1-bucket\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucketAgg\",\"params\":{\"filters\":[{\"input\":{\"query\":\"event:\\\"Send exception\\\"\"},\"label\":\"\"}]}},\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{\"customLabel\":\"Exceptions\"}},\"customLabel\":\"Exceptions\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Time\"}}]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "7e8996e0-c23d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", + "_source": { + "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", + "timeRestore": false, + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + }, + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":21,\"w\":48,\"h\":13,\"i\":\"1\"},\"id\":\"3ade53d0-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":27,\"i\":\"2\"},\"id\":\"9397f350-c23e-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"search\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"3\"},\"id\":\"1ec6a660-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"4\"},\"id\":\"65918380-c244-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"version\":\"6.4.0\"}]", + "title": "Monitoring", + "version": 1 + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "31ac2d60-c25b-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "List of users", + "visState": "{\"title\":\"List of users\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"sum_bucket\",\"schema\":\"metric\",\"params\":{\"customBucket\":{\"id\":\"3-bucket\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"bucketAgg\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},\"customMetric\":{\"id\":\"3-metric\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metricAgg\",\"params\":{}},\"customLabel\":\"Activity\"}},{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"working time\",\"customLabel\":\"Working Time (h)\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "1ec6a660-c244-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Duration of events", + "visState": "{\"title\":\"Duration of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\",\"customLabel\":\"\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"duration\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"exists\":{\"field\":\"duration\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"key\":\"duration\",\"negate\":false,\"type\":\"exists\",\"value\":\"exists\"}}]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "ec510550-c238-11e8-8e1b-758ef07f6de8", + "_type": "index-pattern", + "_source": { + "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"application\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"application.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"box count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"duration\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"event\",\"type\":\"string\",\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"event.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"frame count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"object count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"points count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polygon count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"polyline count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"task\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"task.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"timestamp\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"track count\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"userid\",\"type\":\"string\",\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"userid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"working time\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]", + "fieldFormatMap": "{\"duration\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asSeconds\"}},\"working time\":{\"id\":\"duration\",\"params\":{\"inputFormat\":\"milliseconds\",\"outputFormat\":\"asHours\"}}}", + "title": "cvat*", + "timeFieldName": "@timestamp" + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "65918380-c244-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Number of events", + "visState": "{\"title\":\"Number of events\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"event.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Action\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "d92524b0-c25c-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Activity of users", + "visState": "{\"title\":\"Activity of users\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"User\",\"terms_field\":\"userid.keyword\",\"terms_size\":\"100\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"cvat*\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1},\"aggs\":[]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "9397f350-c23e-11e8-8e1b-758ef07f6de8", + "_type": "search", + "_source": { + "title": "Table with exceptions", + "description": "", + "hits": 0, + "columns": [ + "task", + "type", + "userid", + "stack" + ], + "sort": [ + "@timestamp", + "desc" + ], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":\"event:\\\"Send exception\\\"\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "b6339c70-c7d4-11e8-a035-258d2bd7d91f", + "_type": "visualization", + "_source": { + "title": "Working calendar", + "visState": "{\"title\":\"Working calendar\",\"type\":\"heatmap\",\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_key\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Users\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}}]}", + "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 8\":\"rgb(165,0,38)\",\"8 - 15\":\"rgb(249,142,82)\",\"15 - 23\":\"rgb(255,255,190)\",\"23 - 30\":\"rgb(135,203,103)\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "22250a40-c25d-11e8-8e1b-758ef07f6de8", + "_type": "dashboard", + "_source": { + "title": "Managment", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":21,\"i\":\"1\"},\"id\":\"31ac2d60-c25b-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":21,\"i\":\"2\"},\"id\":\"543f6260-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":21,\"w\":24,\"h\":16,\"i\":\"3\"},\"id\":\"d92524b0-c25c-11e8-8e1b-758ef07f6de8\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.4.0\"},{\"gridData\":{\"x\":0,\"y\":21,\"w\":24,\"h\":16,\"i\":\"4\"},\"version\":\"6.4.0\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"id\":\"b6339c70-c7d4-11e8-a035-258d2bd7d91f\",\"embeddableConfig\":{}}]", + "optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "543f6260-c25c-11e8-8e1b-758ef07f6de8", + "_type": "visualization", + "_source": { + "title": "Working day", + "visState": "{\"title\":\"Working day\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"min\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"Start\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"split\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"_\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"userid.keyword\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"User\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"@timestamp\",\"customLabel\":\"End\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"ec510550-c238-11e8-8e1b-758ef07f6de8\",\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + } +] \ No newline at end of file diff --git a/analytics/kibana/kibana.yml b/analytics/kibana/kibana.yml new file mode 100644 index 000000000000..29f931b4230b --- /dev/null +++ b/analytics/kibana/kibana.yml @@ -0,0 +1,4 @@ +server.host: 0.0.0.0 +elasticsearch.url: http://elasticsearch:9200 +elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ] +kibana.defaultAppId: "discover" diff --git a/analytics/kibana/setup.py b/analytics/kibana/setup.py new file mode 100644 index 000000000000..f1877ffec07c --- /dev/null +++ b/analytics/kibana/setup.py @@ -0,0 +1,40 @@ +#/usr/bin/env python + +import os +import argparse +import requests +import json + +def import_resources(host, port, cfg_file): + with open(cfg_file, 'r') as f: + for saved_object in json.load(f): + _id = saved_object["_id"] + _type = saved_object["_type"] + _doc = saved_object["_source"] + import_saved_object(host, port, _type, _id, _doc) + +def import_saved_object(host, port, _type, _id, data): + saved_objects_api = "http://{}:{}/api/saved_objects/{}/{}".format( + host, port, _type, _id) + request = requests.get(saved_objects_api) + if request.status_code == 404: + print("Creating {} as {}".format(_type, _id)) + request = requests.post(saved_objects_api, json={"attributes": data}, + headers={'kbn-xsrf': 'true'}) + else: + print("Updating {} named {}".format(_type, _id)) + request = requests.put(saved_objects_api, json={"attributes": data}, + headers={'kbn-xsrf': 'true'}) + request.raise_for_status() + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='import Kibana 6.x resources', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('export_file', metavar='FILE', + help='JSON export file with resources') + parser.add_argument('-p', '--port', metavar='PORT', default=5601, type=int, + help='port of Kibana instance') + parser.add_argument('-H', '--host', metavar='HOST', default='kibana', + help='host of Kibana instance') + args = parser.parse_args() + import_resources(args.host, args.port, args.export_file) diff --git a/analytics/logstash/Dockerfile b/analytics/logstash/Dockerfile new file mode 100644 index 000000000000..ad012ccfc8fa --- /dev/null +++ b/analytics/logstash/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION +FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION} +RUN logstash-plugin install logstash-input-http logstash-filter-aggregate \ + logstash-filter-prune logstash-output-email + +COPY logstash.conf /usr/share/logstash/pipeline/ +EXPOSE 5000 diff --git a/analytics/logstash/logstash.conf b/analytics/logstash/logstash.conf new file mode 100644 index 000000000000..f0ecb4caf271 --- /dev/null +++ b/analytics/logstash/logstash.conf @@ -0,0 +1,85 @@ +input { + tcp { + port => 5000 + codec => json + } +} + +filter { + if [logger_name] =~ /cvat.client/ { + # 1. Decode the event from json in 'message' field + # 2. Remove unnecessary field from it + # 3. Type it as client + json { + source => "message" + } + + date { + match => ["timestamp", "UNIX", "UNIX_MS"] + remove_field => "timestamp" + } + + if [event] == "Send exception" { + aggregate { + task_id => "%{userid}_%{application}_%{message}_%{filename}_%{line}" + code => " + require 'time' + + map['userid'] ||= event.get('userid'); + map['application'] ||= event.get('application'); + map['message'] ||= event.get('message'); + map['filename'] ||= event.get('filename'); + map['line'] ||= event.get('line'); + map['task'] ||= event.get('task'); + + map['error_count'] ||= 0; + map['error_count'] += 1; + + map['aggregated_message'] ||= ''; + time = Time.strptime(event.get('timestamp').to_s,'%Q').localtime('+03:00') + map['aggregated_message'] += time.to_s + '\n' + event.get('stack') + '\n\n\n';" + + timeout => 3600 + timeout_tags => ['send_email_notification'] + push_map_as_event_on_timeout => true + } + } + + prune { + blacklist_names => ["level", "host", "logger_name", "message", "path", + "port", "stack_info"] + } + + mutate { + replace => { "type" => "client" } + } + } else if [logger_name] =~ /cvat.server/ { + # 1. Remove unnecessary field from it + # 2. Type it as server + prune { + blacklist_names => ["host", "port"] + } + + mutate { + replace => { "type" => "server" } + } + } +} + +output { + stdout { + codec => rubydebug + } + + if [type] == "client" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "cvat.client" + } + } else if [type] == "server" { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "cvat.server" + } + } +} diff --git a/cvat/apps/engine/logging.py b/cvat/apps/engine/logging.py index 8948821f529b..1cf774b8404e 100644 --- a/cvat/apps/engine/logging.py +++ b/cvat/apps/engine/logging.py @@ -1,19 +1,28 @@ - # Copyright (C) 2018 Intel Corporation # # SPDX-License-Identifier: MIT import os -import inspect import logging from . import models from cvat.settings.base import LOGGING +from cvat.apps.engine.models import Job, Task + +def _get_task(tid): + try: + return Task.objects.get(pk=tid) + except Exception: + raise Exception('{} key must be a task identifier'.format(tid)) +def _get_job(jid): + try: + return models.Job.objects.select_related("segment__task").get(id=jid) + except Exception: + raise Exception('{} key must be a job identifier'.format(jid)) class TaskLoggerStorage: def __init__(self): self._storage = dict() - self._formatter = logging.getLogger('task') def __getitem__(self, tid): if tid not in self._storage: @@ -21,33 +30,13 @@ def __getitem__(self, tid): return self._storage[tid] def _create_task_logger(self, tid): - task = self._get_task(tid) - if task is not None: - configuration = LOGGING.copy() - handler_configuration = configuration['handlers']['file'] - handler_configuration['filename'] = task.get_log_path() - configuration['handlers'] = { - 'file_{}'.format(tid): handler_configuration - } - configuration['loggers'] = { - 'task_{}'.format(tid): { - 'handlers': ['file_{}'.format(tid)], - 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), - } - } - - logging.config.dictConfig(configuration) - logger = logging.getLogger('task_{}'.format(tid)) - return logger - else: - raise Exception('Key must be task indentificator') - - def _get_task(self, tid): - try: - return models.Task.objects.get(pk=tid) - except Exception: - return None + task = _get_task(tid) + + logger = logging.getLogger('cvat.server.task_{}'.format(tid)) + server_file = logging.FileHandler(filename=task.get_log_path()) + logger.addHandler(server_file) + return logger class JobLoggerStorage: def __init__(self): @@ -59,17 +48,41 @@ def __getitem__(self, jid): return self._storage[jid] def _get_task_logger(self, jid): - job = self._get_job(jid) - if job is not None: - return task_logger[job.segment.task.id] - else: - raise Exception('Key must be job identificator') - - def _get_job(self, jid): - try: - return models.Job.objects.select_related("segment__task").get(id=jid) - except Exception: - return None + job = _get_job(jid) + return task_logger[job.segment.task.id] + +class TaskClientLoggerStorage: + def __init__(self): + self._storage = dict() + + def __getitem__(self, tid): + if tid not in self._storage: + self._storage[tid] = self._create_client_logger(tid) + return self._storage[tid] + + def _create_client_logger(self, tid): + task = _get_task(tid) + logger = logging.getLogger('cvat.client.task_{}'.format(tid)) + client_file = logging.FileHandler(filename=task.get_client_log_path()) + logger.addHandler(client_file) + + return logger + +class JobClientLoggerStorage: + def __init__(self): + self._storage = dict() + + def __getitem__(self, jid): + if jid not in self._storage: + self._storage[jid] = self._get_task_logger(jid) + return self._storage[jid] + + def _get_task_logger(self, jid): + job = _get_job(jid) + return task_client_logger[job.segment.task.id] task_logger = TaskLoggerStorage() job_logger = JobLoggerStorage() +global_logger = logging.getLogger('cvat.server') +job_client_logger = JobClientLoggerStorage() +task_client_logger = TaskClientLoggerStorage() \ No newline at end of file diff --git a/cvat/apps/engine/static/engine/js/logger.js b/cvat/apps/engine/static/engine/js/logger.js index 45bf95bfc181..e26aef9de768 100644 --- a/cvat/apps/engine/static/engine/js/logger.js +++ b/cvat/apps/engine/static/engine/js/logger.js @@ -80,7 +80,7 @@ var LoggerHandler = function(applicationName, jobId) return new Promise( (resolve, reject) => { let xhr = new XMLHttpRequest(); - xhr.open('POST', '/logs/exception/' + this._jobId); + xhr.open('POST', '/save/exception/' + this._jobId); xhr.setRequestHeader('Content-Type', 'application/json'); xhr.setRequestHeader("X-CSRFToken", Cookies.get('csrftoken')); @@ -202,25 +202,31 @@ var LoggerHandler = function(applicationName, jobId) /* -Log message has simple json format - each message is set of "key" : "value" pairs inside curly braces - {"key1" : "string_value", "key2" : number_value, ...} -Value may be string or number (see json spec) -required fields for all event types: +Log message has simple json format - each message is set of "key" : "value" +pairs inside curly braces - {"key1" : "string_value", "key2" : number_value, +...} Value may be string or number (see json spec) required fields for all event +types: NAME TYPE DESCRIPTION "event" string see EventType enum description of possible values. -"timestamp" number timestamp in UNIX format - the number of seconds or milliseconds that have elapsed since 00:00:00 Thursday, 1 January 1970 +"timestamp" number timestamp in UNIX format - the number of seconds + or milliseconds that have elapsed since 00:00:00 + Thursday, 1 January 1970 "application" string application name "userid" string Unique userid "task" string Unique task id. (Is expected corresponding Jira task id) -"count" is requiered field for "Add object", "Delete object", "Copy track", "Propagate object", "Merge objecrs", "Undo action" and "Redo action" -events with number value. +"count" is requiered field for "Add object", "Delete object", "Copy track", +"Propagate object", "Merge objecrs", "Undo action" and "Redo action" events with +number value. -Example : { "event" : "Add object", "timestamp" : 1486040342867, "application" : "CVAT", "duration" : 4200, "userid" : "ESAZON1X-MOBL", "count" : 1, "type" : "bounding box" } +Example : { "event" : "Add object", "timestamp" : 1486040342867, "application" : +"CVAT", "duration" : 4200, "userid" : "ESAZON1X-MOBL", "count" : 1, "type" : +"bounding box" } -Types of supported events. -Minimum subset of events to generate simple report are Logger.EventType.addObject, Logger.EventType.deleteObject and Logger.EventType.sendTaskInfo. -Value of "count" property should be a number. +Types of supported events. Minimum subset of events to generate simple report +are Logger.EventType.addObject, Logger.EventType.deleteObject and +Logger.EventType.sendTaskInfo. Value of "count" property should be a number. */ var Logger = { @@ -276,50 +282,67 @@ var Logger = { EventType: { // dumped as "Paste object". There are no additional required fields. pasteObject: 0, - // dumped as "Change attribute". There are no additional required fields. + // dumped as "Change attribute". There are no additional required + // fields. changeAttribute: 1, // dumped as "Drag object". There are no additional required fields. dragObject: 2, - // dumped as "Delete object". "count" is required field, value of deleted objects should be positive number. + // dumped as "Delete object". "count" is required field, value of + // deleted objects should be positive number. deleteObject: 3, // dumped as "Press shortcut". There are no additional required fields. pressShortcut: 4, // dumped as "Resize object". There are no additional required fields. resizeObject: 5, - // dumped as "Send logs". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Send logs". It's expected that event has "duration" field, + // but it isn't necessary. sendLogs: 6, - // dumped as "Save job". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Save job". It's expected that event has "duration" field, + // but it isn't necessary. saveJob: 7, // dumped as "Jump frame". There are no additional required fields. jumpFrame: 8, - // dumped as "Draw object". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Draw object". It's expected that event has "duration" + // field, but it isn't necessary. drawObject: 9, // dumped as "Change label". changeLabel: 10, - // dumped as "Send task info". "track count", "frame count", "object count" are required fields. It's expected that event has "current_frame" field. + // dumped as "Send task info". "track count", "frame count", "object + // count" are required fields. It's expected that event has + // "current_frame" field. sendTaskInfo: 11, - // dumped as "Load job". "track count", "frame count", "object count" are required fields. It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Load job". "track count", "frame count", "object count" + // are required fields. It's expected that event has "duration" field, + // but it isn't necessary. loadJob: 12, - // dumped as "Move image". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Move image". It's expected that event has "duration" + // field, but it isn't necessary. moveImage: 13, - // dumped as "Zoom image". It's expected that event has "duration" field, but it isn't necessary. + // dumped as "Zoom image". It's expected that event has "duration" + // field, but it isn't necessary. zoomImage: 14, // dumped as "Lock object". There are no additional required fields. lockObject: 15, - // dumped as "Merge objects". "count" is required field with positive or negative number value. + // dumped as "Merge objects". "count" is required field with positive or + // negative number value. mergeObjects: 16, // dumped as "Copy object". "count" is required field with number value. copyObject: 17, - // dumped as "Propagate object". "count" is required field with number value. + // dumped as "Propagate object". "count" is required field with number + // value. propagateObject: 18, - // dumped as "Undo action". "count" is required field with positive or negative number value. + // dumped as "Undo action". "count" is required field with positive or + // negative number value. undoAction: 19, - // dumped as "Redo action". "count" is required field with positive or negative number value. + // dumped as "Redo action". "count" is required field with positive or + // negative number value. redoAction: 20, - // dumped as "Send user activity". "working_time" is required field with positive number value. + // dumped as "Send user activity". "working_time" is required field with + // positive number value. sendUserActivity: 21, - // dumped as "Send exception". Use to send any exception events to the server. - // "message", "filename", "line" are mandatory fields. "stack" and "column" are optional. + // dumped as "Send exception". Use to send any exception events to the + // server. "message", "filename", "line" are mandatory fields. "stack" + // and "column" are optional. sendException: 22, // dumped as "Change frame". There are no additional required fields. changeFrame: 23, @@ -356,10 +379,12 @@ var Logger = { /** * Logger.addContinuedEvent Use to add log event with duration field. - * Duration will be calculated automatically when LogEvent.close() method of returned Object will be called. - * Note: in case of LogEvent.close() method will not be callsed event will not be sended to server + * Duration will be calculated automatically when LogEvent.close() method of + * returned Object will be called. Note: in case of LogEvent.close() method + * will not be callsed event will not be sent to server * @param {Logger.EventType} type Event Type - * @param {Object} values Any event values, for example {count: 1, label: 'vehicle'} + * @param {Object} values Any event values, for example {count: 1, label: + * 'vehicle'} * @return {LogEvent} instance of LogEvent * @static */ @@ -370,7 +395,8 @@ var Logger = { /** * Logger.shortkeyLogDecorator use for decorating the shortkey handlers. - * This decorator just create appropriate log event and close it when decored function will performed. + * This decorator just create appropriate log event and close it when + * decored function will performed. * @param {Function} decoredFunc is function for decorating * @return {Function} is decorated decoredFunc * @static @@ -387,7 +413,7 @@ var Logger = { }, /** - * Logger.sendLogs Try to send exception logs to the server immediatly. + * Logger.sendLogs Try to send exception logs to the server immediately. * @return {Promise} * @param {LogEvent} exceptionEvent * @static @@ -414,7 +440,8 @@ var Logger = { }, /** - * Logger.setUsername just set username property which will be added to all log messages + * Logger.setUsername just set username property which will be added to all + * log messages * @param {String} username * @static */ @@ -423,7 +450,8 @@ var Logger = { this._logger.setUsername(username); }, - /** Logger.updateUserActivityTimer method updates internal timer for working time calculation logic + /** Logger.updateUserActivityTimer method updates internal timer for working + * time calculation logic * @static */ updateUserActivityTimer: function() @@ -431,11 +459,12 @@ var Logger = { this._logger.updateTimer(); }, - /** Logger.setTimeThreshold set time threshold in ms for EventType. - * If time interval betwwen incoming log events less than threshold events will be collapsed. - * Note that result event will have timestamp of first event, - * In case of time threshold used for continued event duration will be difference between - * first and last event timestamps and other fields from last event. + /** Logger.setTimeThreshold set time threshold in ms for EventType. If time + * interval betwwen incoming log events less than threshold events will be + * collapsed. Note that result event will have timestamp of first event, In + * case of time threshold used for continued event duration will be + * difference between first and last event timestamps and other fields from + * last event. * @static * @param {Logger.EventType} eventType * @param {Number} threshold @@ -445,7 +474,8 @@ var Logger = { this._logger.setTimeThreshold(eventType, threshold); }, - /** Logger._eventTypeToString private method to transform Logger.EventType to string + /** Logger._eventTypeToString private method to transform Logger.EventType + * to string * @param {Logger.EventType} type Event Type * @return {String} string reppresentation of Logger.EventType * @static diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index f6d755be5923..782450bc4cba 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -30,9 +30,7 @@ from distutils.dir_util import copy_tree from . import models -from .logging import task_logger, job_logger - -global_logger = logging.getLogger(__name__) +from .logging import task_logger, job_logger, global_logger ############################# Low Level server API diff --git a/cvat/apps/engine/urls.py b/cvat/apps/engine/urls.py index cc1e3496cfa7..218f0411fb7f 100644 --- a/cvat/apps/engine/urls.py +++ b/cvat/apps/engine/urls.py @@ -22,4 +22,5 @@ path('save/annotation/task/', views.save_annotation_for_task), path('get/annotation/job/', views.get_annotation), path('get/username', views.get_username), + path('save/exception/', views.catch_client_exception) ] diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index f1edac086645..3caae2809683 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -5,7 +5,6 @@ import os import json -import logging import traceback from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse @@ -18,13 +17,20 @@ from . import annotation, task, models from cvat.settings.base import JS_3RDPARTY from cvat.apps.authentication.decorators import login_required -from cvat.apps.log_proxy.proxy_logger import client_log_proxy from requests.exceptions import RequestException -from .logging import task_logger, job_logger - -global_logger = logging.getLogger(__name__) +import logging +from .logging import task_logger, job_logger, global_logger, job_client_logger ############################# High Level server API +@login_required +@permission_required('engine.view_task', raise_exception=True) +def catch_client_exception(request, jid): + data = json.loads(request.body.decode('utf-8')) + for event in data['exceptions']: + job_client_logger[jid].error(json.dumps(event)) + + return HttpResponse() + @login_required def dispatch_request(request): """An entry point to dispatch legacy requests""" @@ -243,7 +249,8 @@ def save_annotation_for_job(request, jid): if 'annotation' in data: annotation.save_job(jid, json.loads(data['annotation'])) if 'logs' in data: - client_log_proxy.push_logs(jid, json.loads(data['logs'])) + for event in json.loads(data['logs']): + job_client_logger[jid].info(json.dumps(event)) except RequestException as e: job_logger[jid].error("cannot send annotation logs for job {}".format(jid), exc_info=True) return HttpResponseBadRequest(str(e)) diff --git a/cvat/apps/log_proxy/__init__.py b/cvat/apps/log_proxy/__init__.py deleted file mode 100644 index d8e62e54b356..000000000000 --- a/cvat/apps/log_proxy/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - diff --git a/cvat/apps/log_proxy/admin.py b/cvat/apps/log_proxy/admin.py deleted file mode 100644 index af8dfc47525b..000000000000 --- a/cvat/apps/log_proxy/admin.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.contrib import admin - -# Register your models here. - diff --git a/cvat/apps/log_proxy/apps.py b/cvat/apps/log_proxy/apps.py deleted file mode 100644 index 6b456281f3b7..000000000000 --- a/cvat/apps/log_proxy/apps.py +++ /dev/null @@ -1,11 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.apps import AppConfig - - -class LogProxyConfig(AppConfig): - name = 'log_proxy' - diff --git a/cvat/apps/log_proxy/migrations/__init__.py b/cvat/apps/log_proxy/migrations/__init__.py deleted file mode 100644 index d8e62e54b356..000000000000 --- a/cvat/apps/log_proxy/migrations/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - diff --git a/cvat/apps/log_proxy/models.py b/cvat/apps/log_proxy/models.py deleted file mode 100644 index cdf3b0827bf1..000000000000 --- a/cvat/apps/log_proxy/models.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.db import models - -# Create your models here. - diff --git a/cvat/apps/log_proxy/proxy_logger.py b/cvat/apps/log_proxy/proxy_logger.py deleted file mode 100644 index d10398c568ff..000000000000 --- a/cvat/apps/log_proxy/proxy_logger.py +++ /dev/null @@ -1,91 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.conf import settings -import os -import logging -import requests -import json -from urllib.parse import urlparse -from enum import Enum -from cvat.apps.engine.models import Job, Task - -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry - -class ClientLoggerStorage: - def __init__(self): - self._storage = dict() - self._formatter = logging.Formatter('%(message)s') - - def __getitem__(self, tid): - if tid not in self._storage: - self._storage[tid] = self._create_client_logger(tid) - return self._storage[tid] - - def _create_client_logger(self, tid): - task = self._get_task(tid) - logger = logging.getLogger(name='client_annotation_logger_{}'.format(tid)) - logger.setLevel(logging.INFO) - handler = logging.FileHandler(filename=task.get_client_log_path()) - handler.setFormatter(self._formatter) - logger.addHandler(handler) - return logger - - def _get_task(self, tid): - try: - return Task.objects.get(pk=tid) - except Exception: - raise Exception('Key must be task indentificator') - -class ClientLogProxy(): - class _HandlerType(Enum): - FILE = 1 - HTTP = 2 - - def __init__(self): - self._client_logger = ClientLoggerStorage() - def file_log_handler(tid, messages): - for event in messages: - self._client_logger[tid].info(json.dumps(event)) - - self._handlers = {self._HandlerType.FILE: file_log_handler} - - log_server_url = os.environ.get('DJANGO_LOG_SERVER_URL') - - def create_retry_session(retries=3, session=None, backoff_factor=0.3): - session = session or requests.Session() - retry = Retry(total=retries, backoff_factor=backoff_factor) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - return session - - if log_server_url: - parse_result = urlparse(log_server_url) - - if parse_result.scheme and 'http' not in parse_result.scheme: - raise Exception('unsuported annotation log destination') - - def http_log_handler(taskID, messages): - r = create_retry_session().post(url=log_server_url, json=messages, verify=False) - r.raise_for_status() - - self._handlers[self._HandlerType.HTTP] = http_log_handler - - def push_logs(self, jid, logs): - taskID = self._get_task_id(jid) - - for handler in self._handlers.values(): - handler(taskID, logs) - - def _get_task_id(self, jid): - try: - job = Job.objects.select_related("segment__task").get(id=jid) - return job.segment.task.id - except: - raise Exception('Key must be job indentificator') - -client_log_proxy = ClientLogProxy() diff --git a/cvat/apps/log_proxy/tests.py b/cvat/apps/log_proxy/tests.py deleted file mode 100644 index 53bc3b7adb85..000000000000 --- a/cvat/apps/log_proxy/tests.py +++ /dev/null @@ -1,9 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.test import TestCase - -# Create your tests here. - diff --git a/cvat/apps/log_proxy/urls.py b/cvat/apps/log_proxy/urls.py deleted file mode 100644 index c483e7fdac66..000000000000 --- a/cvat/apps/log_proxy/urls.py +++ /dev/null @@ -1,12 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.urls import path -from . import views - -urlpatterns = [ - path('exception/', views.exception_receiver), -] - diff --git a/cvat/apps/log_proxy/views.py b/cvat/apps/log_proxy/views.py deleted file mode 100644 index 645c20362499..000000000000 --- a/cvat/apps/log_proxy/views.py +++ /dev/null @@ -1,25 +0,0 @@ - -# Copyright (C) 2018 Intel Corporation -# -# SPDX-License-Identifier: MIT - -from django.http import HttpResponse, HttpResponseBadRequest -from django.contrib.auth.decorators import permission_required -from .proxy_logger import client_log_proxy -from cvat.apps.authentication.decorators import login_required - - -import json - -# Create your views here. -@login_required() -@permission_required('engine.view_task', raise_exception=True) -def exception_receiver(request, jid): - data = json.loads(request.body.decode('utf-8')) - try: - if 'exceptions' in data: - client_log_proxy.push_logs(jid, data['exceptions']) - except Exception as e: - return HttpResponseBadRequest(str(e)) - - return HttpResponse() diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 8f6092361d2b..839d118833a3 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -22,3 +22,4 @@ scipy==1.0.1 sqlparse==0.2.4 django-sendfile==0.3.11 dj-pagination==2.3.2 +python-logstash==0.4.6 diff --git a/cvat/settings/base.py b/cvat/settings/base.py index cb937f473e22..1506625343ee 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -187,23 +187,40 @@ 'class': 'logging.StreamHandler', 'formatter': 'standard', }, - 'file': { + 'server_file': { 'class': 'logging.handlers.RotatingFileHandler', - 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), - 'filename': os.path.join(BASE_DIR, 'logs', 'cvat.log'), + 'level': 'DEBUG', + 'filename': os.path.join(BASE_DIR, 'logs', 'cvat_server.log'), 'formatter': 'standard', 'maxBytes': 1024*1024*50, # 50 MB 'backupCount': 5, + }, + 'logstash': { + 'level': 'INFO', + 'class': 'logstash.TCPLogstashHandler', + 'host': os.getenv('DJANGO_LOG_SERVER_HOST', 'localhost'), + 'port': os.getenv('DJANGO_LOG_SERVER_PORT', 5000), + 'version': 1, + 'message_type': 'django', } }, 'loggers': { - 'cvat': { - 'handlers': ['console', 'file'], + 'cvat.server': { + 'handlers': ['console', 'server_file'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), + }, + + 'cvat.client': { + 'handlers': [], 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), } }, } +if os.getenv('DJANGO_LOG_SERVER_HOST'): + LOGGING['loggers']['cvat.server']['handlers'] += ['logstash'] + LOGGING['loggers']['cvat.client']['handlers'] += ['logstash'] + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ diff --git a/cvat/urls.py b/cvat/urls.py index 657af2f45e4d..0d27ee7ab02f 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -30,8 +30,7 @@ path('dashboard/', include('cvat.apps.dashboard.urls')), path('django-rq/', include('django_rq.urls')), path('auth/', include('cvat.apps.authentication.urls')), - path('documentation/', include('cvat.apps.documentation.urls')), - path('logs/', include('cvat.apps.log_proxy.urls')) + path('documentation/', include('cvat.apps.documentation.urls')) ] if 'yes' == os.environ.get('TF_ANNOTATION', 'no'): diff --git a/docker-compose.yml b/docker-compose.yml index 490684f0f63b..d55f64b4bd22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,10 @@ services: cvat_db: container_name: cvat_db image: postgres:10.3-alpine + networks: + default: + aliases: + - db restart: always environment: POSTGRES_USER: root @@ -19,6 +23,10 @@ services: cvat_redis: container_name: cvat_redis image: redis:4.0.5-alpine + networks: + default: + aliases: + - redis restart: always cvat: @@ -42,7 +50,8 @@ services: WITH_TESTS: "no" environment: DJANGO_MODWSGI_EXTRA_ARGS: "" - DJANGO_LOG_SERVER_URL: "" + DJANGO_LOG_SERVER_HOST: "" + DJANGO_LOG_SERVER_PORT: "" volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys diff --git a/supervisord.conf b/supervisord.conf index 70c3cf8dee38..87020e5b8a09 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -18,16 +18,19 @@ pidfile=/tmp/supervisord/supervisord.pid ; pidfile location childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live [program:rqworker_default] -command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic "/usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" +command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic \ + "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 default" numprocs=2 process_name=rqworker_default_%(process_num)s [program:rqworker_low] -command=%(ENV_HOME)s/wait-for-it.sh cvat_redis:6379 -t 0 -- bash -ic "/usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" +command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ + "exec /usr/bin/python3 %(ENV_HOME)s/manage.py rqworker -v 3 low" numprocs=1 [program:runserver] -command=%(ENV_HOME)s/wait-for-it.sh cvat_db:5432 -t 0 -- bash -ic "/usr/bin/python3 ~/manage.py migrate && \ - exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ +command=%(ENV_HOME)s/wait-for-it.sh db:5432 -t 0 -- bash -ic \ + "/usr/bin/python3 ~/manage.py migrate && \ + exec /usr/bin/python3 $HOME/manage.py runmodwsgi --log-to-terminal --port 8080 \ --limit-request-body 1073741824 --log-level INFO --include-file ~/mod_wsgi.conf \ %(ENV_DJANGO_MODWSGI_EXTRA_ARGS)s --locale %(ENV_LC_ALL)s"