Audit logs (cvat-ai#5646)

retailnext · Jul 1, 2023 · 6d25648 · 6d25648
1 parent 471d068
commit 6d25648
Show file tree

Hide file tree

Showing 81 changed files with 3,686 additions and 2,193 deletions.
diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml
@@ -35,24 +35,6 @@ jobs:
             ${{ runner.os }}-build-ui-${{ needs.get-sha.outputs.sha }}
             ${{ runner.os }}-build-ui-
 
-      - uses: actions/cache@v3
-        id: elasticsearch-cache-action
-        with:
-          path: /tmp/cvat_cache_elasticsearch
-          key: ${{ runner.os }}-build-elasticsearch-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-build-elasticsearch-${{ needs.get-sha.outputs.sha }}
-            ${{ runner.os }}-build-elasticsearch-
-
-      - uses: actions/cache@v3
-        id: logstash-cache-action
-        with:
-          path: /tmp/cvat_cache_logstash
-          key: ${{ runner.os }}-build-logstash-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-build-logstash-${{ needs.get-sha.outputs.sha }}
-            ${{ runner.os }}-build-logstash-
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
@@ -72,34 +54,10 @@ jobs:
           cache-from: type=local,src=/tmp/cvat_cache_ui
           cache-to: type=local,dest=/tmp/cvat_cache_ui-new
 
-      - name: Caching CVAT Elasticsearch
-        uses: docker/build-push-action@v2
-        with:
-          context: ./components/analytics/elasticsearch/
-          file: ./components/analytics/elasticsearch/Dockerfile
-          cache-from: type=local,src=/tmp/cvat_cache_elasticsearch
-          cache-to: type=local,dest=/tmp/cvat_cache_elasticsearch-new
-          build-args: ELK_VERSION=6.8.23
-
-      - name: Caching CVAT Logstash
-        uses: docker/build-push-action@v2
-        with:
-          context: ./components/analytics/logstash/
-          file: ./components/analytics/logstash/Dockerfile
-          cache-from: type=local,src=/tmp/cvat_cache_logstash
-          cache-to: type=local,dest=/tmp/cvat_cache_logstash-new
-          build-args: ELK_VERSION=6.8.23
-
       - name: Moving cache
         run: |
           rm -rf /tmp/cvat_cache_server
           mv /tmp/cvat_cache_server-new /tmp/cvat_cache_server
 
           rm -rf /tmp/cvat_cache_ui
           mv /tmp/cvat_cache_ui-new /tmp/cvat_cache_ui
-
-          rm -rf /tmp/cvat_cache_elasticsearch
-          mv /tmp/cvat_cache_elasticsearch-new /tmp/cvat_cache_elasticsearch
-
-          rm -rf /tmp/cvat_cache_logstash
-          mv /tmp/cvat_cache_logstash-new /tmp/cvat_cache_logstash
diff --git a/.github/workflows/full.yml b/.github/workflows/full.yml
@@ -115,38 +115,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@master
 
-      - name: Getting CVAT Elasticsearch cache from the default branch
-        uses: actions/cache@v3
-        with:
-          path: /tmp/cvat_cache_elasticsearch
-          key: ${{ runner.os }}-build-elasticsearch-${{ needs.search_cache.outputs.sha }}
-
-      - name: Getting CVAT Logstash cache from the default branch
-        uses: actions/cache@v3
-        with:
-          path: /tmp/cvat_cache_logstash
-          key: ${{ runner.os }}-build-logstash-${{ needs.search_cache.outputs.sha }}
-
-      - name: Building CVAT Elasticsearch
-        uses: docker/build-push-action@v2
-        with:
-          context: ./components/analytics/elasticsearch/
-          file: ./components/analytics/elasticsearch/Dockerfile
-          cache-from: type=local,src=/tmp/cvat_cache_elasticsearch
-          tags: cvat_elasticsearch:latest
-          load: true
-          build-args: ELK_VERSION=6.8.23
-
-      - name: Building CVAT Logstash
-        uses: docker/build-push-action@v2
-        with:
-          context: ./components/analytics/logstash/
-          file: ./components/analytics/logstash/Dockerfile
-          cache-from: type=local,src=/tmp/cvat_cache_logstash
-          tags: cvat_logstash:latest
-          load: true
-          build-args: ELK_VERSION=6.8.23
-
       - name: Download CVAT server image
         uses: actions/download-artifact@v3
         with:

diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml
@@ -129,18 +129,6 @@ jobs:
           path: /tmp/cvat_cache_ui
           key: ${{ runner.os }}-build-ui-${{ needs.search_cache.outputs.sha }}
 
-      - name: Getting CVAT Logstash cache from the default branch
-        uses: actions/cache@v3
-        with:
-          path: /tmp/cvat_cache_logstash
-          key: ${{ runner.os }}-build-logstash-${{ needs.search_cache.outputs.sha }}
-
-      - name: Getting CVAT Elasticsearch cache from the default branch
-        uses: actions/cache@v3
-        with:
-          path: /tmp/cvat_cache_elasticsearch
-          key: ${{ runner.os }}-build-elasticsearch-${{ needs.search_cache.outputs.sha }}
-
       - name: Building CVAT UI image
         uses: docker/build-push-action@v2
         with:
@@ -150,26 +138,6 @@ jobs:
           tags: cvat/ui:latest
           load: true
 
-      - name: Building CVAT Logstash image
-        uses: docker/build-push-action@v2
-        with:
-          context: ./components/analytics/logstash/
-          file: ./components/analytics/logstash/Dockerfile
-          cache-from: type=local,src=/tmp/cvat_cache_logstash
-          build-args: ELK_VERSION=6.8.23
-          tags: cvat_logstash
-          load: true
-
-      - name: Building CVAT Elasticsearch image
-        uses: docker/build-push-action@v2
-        with:
-          context: ./components/analytics/elasticsearch/
-          file: ./components/analytics/elasticsearch/Dockerfile
-          cache-from: type=local,src=/tmp/cvat_cache_elasticsearch
-          build-args: ELK_VERSION=6.8.23
-          tags: cvat_elasticsearch
-          load: true
-
       - name: CVAT server. Extract metadata (tags, labels) for Docker
         id: meta-server
         uses: docker/metadata-action@master

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -107,7 +107,9 @@
             "env": {
                 "CVAT_SERVERLESS": "1",
                 "ALLOWED_HOSTS": "*",
-                "IAM_OPA_BUNDLE": "1"
+                "IAM_OPA_BUNDLE": "1",
+                "DJANGO_LOG_SERVER_HOST": "localhost",
+                "DJANGO_LOG_SERVER_PORT": "8282"
             },
             "args": [
                 "runserver",
@@ -144,11 +146,14 @@
                 "rqworker",
                 "import",
                 "--worker-class",
-                "cvat.rqworker.SimpleWorker",
+                "cvat.rqworker.SimpleWorker"
             ],
             "django": true,
             "cwd": "${workspaceFolder}",
-            "env": {},
+            "env": {
+                "DJANGO_LOG_SERVER_HOST": "localhost",
+                "DJANGO_LOG_SERVER_PORT": "8282"
+            },
             "console": "internalConsole"
         },
         {
@@ -167,7 +172,10 @@
             ],
             "django": true,
             "cwd": "${workspaceFolder}",
-            "env": {},
+            "env": {
+                "DJANGO_LOG_SERVER_HOST": "localhost",
+                "DJANGO_LOG_SERVER_PORT": "8282"
+            },
             "console": "internalConsole"
         },
         {

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -26,6 +26,7 @@ Tracks can be exported/imported to/from Datumaro and Sly Pointcloud formats (<ht
 - Dummy github and google authentication servers (<https://github.com/opencv/cvat/pull/5444>)
 - \[Server API\] Simple filters for object collection endpoints
   (<https://github.com/opencv/cvat/pull/5575>)
+- Analytics based on Clickhouse, Vector and Grafana instead of the ELK stack (<https://github.com/opencv/cvat/pull/5646>)
 
 ### Changed
 - The Docker Compose files now use the Compose Specification version

diff --git a/components/analytics/clickhouse/init.sh b/components/analytics/clickhouse/init.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+CLICKHOUSE_DB="${CLICKHOUSE_DB:-cvat}";
+CLICKHOUSE_USER="${CLICKHOUSE_USER:-user}";
+CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-password}";
+
+cat <<EOT > /etc/clickhouse-server/users.d/user.xml
+<yandex>
+  <!-- Docs: <https://clickhouse.tech/docs/en/operations/settings/settings_users/> -->
+  <users>
+    <${CLICKHOUSE_USER}>
+      <profile>default</profile>
+      <networks>
+        <ip>::/0</ip>
+      </networks>
+      <password>${CLICKHOUSE_PASSWORD}</password>
+      <quota>default</quota>
+    </${CLICKHOUSE_USER}>
+  </users>
+</yandex>
+EOT
+
+clickhouse-client --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DB}";
+
+echo "
+CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_DB}.events
+(
+    \`scope\` String NOT NULL,
+    \`obj_name\` String NULL,
+    \`obj_id\` UInt64 NULL,
+    \`obj_val\` String NULL,
+    \`source\` String NOT NULL,
+    \`timestamp\` DateTime64(3, 'Etc/UTC') NOT NULL,
+    \`count\` UInt16 NULL,
+    \`duration\` UInt32 DEFAULT toUInt32(0),
+    \`project_id\` UInt64 NULL,
+    \`task_id\` UInt64 NULL,
+    \`job_id\` UInt64 NULL,
+    \`user_id\` UInt64 NULL,
+    \`user_name\` String NULL,
+    \`user_email\` String NULL,
+    \`org_id\` UInt64 NULL,
+    \`org_slug\` String NULL,
+    \`payload\` String NULL
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(timestamp)
+ORDER BY (timestamp)
+SETTINGS index_granularity = 8192
+;" | clickhouse-client
diff --git a/components/analytics/docker-compose.analytics.yml b/components/analytics/docker-compose.analytics.yml
diff --git a/components/analytics/elasticsearch/Dockerfile b/components/analytics/elasticsearch/Dockerfile
diff --git a/components/analytics/elasticsearch/elasticsearch.yml b/components/analytics/elasticsearch/elasticsearch.yml