# This is a combination of 2 commits.

# This is the 1st commit message:

# This is a combination of 3 commits.
# This is the 1st commit message:

# This is a combination of 8 commits.
# This is the 1st commit message:

setup config

Add built from source code

one domain name insted of two

add installation guide

update gitignore

# This is the commit message cvat-ai#2:

delete serverless (nuclio)

# This is the commit message cvat-ai#3:

add backup script to gitignore

# This is the commit message cvat-ai#4:

Add script for backup data

# This is the commit message cvat-ai#5:

Add docstrings for backup script

# This is the commit message cvat-ai#6:

Update backup script to delete after 30 days

# This is the commit message cvat-ai#7:

Fix backup error

# This is the commit message cvat-ai#8:

Update backup rules, and decrease cache size from 1 Tb to 10 Gb

# This is the commit message cvat-ai#2:

one domain name insted of two

# This is the commit message cvat-ai#3:

add installation guide

# This is the commit message cvat-ai#2:

delete serverless (nuclio)

.gitignore

@@ -13,6 +13,7 @@
 node_modules
 /Mask_RCNN/
 /letsencrypt-webroot/
+/backup/
 
 # Ignore temporary files
 docker-compose.override.yml
@@ -34,7 +35,6 @@ yarn-error.log*
 #Ignore Cypress tests temp files
 /tests/cypress/fixtures
 /tests/cypress/screenshots
-.idea/
 
 #Ignore helm-related files
 /helm-chart/Chart.lock
@@ -46,4 +46,5 @@ yarn-error.log*
 /site/public/
 /site/resources/
 /site/node_modules/
-/site/tech-doc-hugo
+/site/tech-doc-hugo
+

backup.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env zsh
+
+# Define directory with current datetime
+DIR=$HOME/cvat/backup/$(date +%Y%m%d_%H%M%S)
+
+# Create this dir if not exist
+mkdir -p $DIR
+
+
+# Stop all containers and get backup data from posgres db with annotation data and kibana data
+docker-compose stop
+mkdir -p backup
+
+docker run --rm --name temp_backup --volumes-from cvat_db -v $DIR:/backup ubuntu tar -cjvf /backup/cvat_db.tar.bz2 /var/lib/postgresql/data
+
+docker run --rm --name temp_backup --volumes-from cvat -v $DIR:/backup ubuntu tar -cjvf /backup/cvat_data.tar.bz2 /home/django/data
+
+docker run --rm --name temp_backup --volumes-from cvat_elasticsearch -v $DIR:/backup ubuntu tar -cjvf /backup/cvat_events.tar.bz2 /usr/share/elasticsearch/data
+
+
+# Run all stoped containers
+docker-compose \
+-f docker-compose.yml \
+-f docker-compose.dev.yml \
+-f docker-compose.override.yml \
+-f components/analytics/docker-compose.analytics.yml \
+up -d --build
+
+# Copy created dir with backup data to min.oi storage
+~/mc cp -r $DIR minio/cvat-backup
+
+# Delete all backups older than 5 days from this server
+find ~/cvat/backup/* -type d -ctime +5 -exec rm -rf {} \;
+
+# Delete all backups older than 30 days from min.io storage
+~/mc rm --recursive --force --older-than 30d minio/cvat-backup
+

components/analytics/elasticsearch/elasticsearch.yml

@@ -1,3 +1,4 @@
 http.host: 0.0.0.0
 script.painless.regex.enabled: true
 path.repo: ['/usr/share/elasticsearch/data/backup']
+cluster.routing.allocation.disk.threshold_enabled: false

components/serverless/docker-compose.serverless.yml

@@ -2,7 +2,7 @@ version: '3.3'
 services:
   nuclio:
     container_name: nuclio
-    image: quay.io/nuclio/dashboard:1.5.16-amd64
+    image: quay.io/nuclio/dashboard:1.6.1-amd64
     restart: always
     networks:
       - cvat

cvat/settings/base.py

@@ -454,8 +454,8 @@ def add_ssh_keys():
     # this setting limits the number of projects for the user
     'project_limit': None,
 
-    # this setting reduces task visibility to owner and assignee only
-    'reduce_task_visibility': False,
+    # this setting reduse task visibility to owner and assignee only
+    'reduce_task_visibility': True,
 
     # allow access to analytics component to users with the following roles
     'analytics_access': (
@@ -473,7 +473,7 @@ def add_ssh_keys():
        'LOCATION' : CACHE_ROOT,
        'TIMEOUT' : None,
        'OPTIONS' : {
-            'size_limit' : 2 ** 40, # 1 Tb
+            'size_limit' : 2 ** 30 * 10, # 10 GB 
        }
    }
 }

cvat_proxy/conf.d/cvat.conf.template

@@ -0,0 +1,49 @@
+server {
+    listen       80;
+    server_name  _ default;
+    return       404;
+}
+
+server {
+    listen       80;
+    server_name  ${CVAT_HOST};
+
+    location ^~ /.well-known/acme-challenge/ {
+      default_type "text/plain";
+      root /var/tmp/letsencrypt-webroot;
+    }
+
+    location / {
+      return 301 https://$server_name$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    server_name ${CVAT_HOST};
+
+    ssl_certificate /etc/ssl/private/${CVAT_HOST}/site.cer;
+    ssl_certificate_key /etc/ssl/private/${CVAT_HOST}/site.key;
+    ssl_trusted_certificate /etc/ssl/private/${CVAT_HOST}/fullchain.cer;
+
+    # security options
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    ssl_prefer_server_ciphers on;
+    ssl_stapling on;
+    ssl_session_timeout 24h;
+    ssl_session_cache shared:SSL:2m;
+    ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!3DES';
+
+    proxy_pass_header       X-CSRFToken;
+    proxy_set_header        Host $http_host;
+    proxy_pass_header       Set-Cookie;
+
+    location ~* /api/.*|git/.*|analytics/.*|static/.*|admin(?:/(.*))?.*|documentation/.*|django-rq(?:/(.*))? {
+        proxy_pass              http://cvat:8080;
+    }
+
+    location / {
+        proxy_pass              http://cvat_ui;
+    }
+}
+

docker-compose.override.yml

@@ -0,0 +1,26 @@
+version: '3.3'
+
+services:
+  cvat_proxy:
+    environment:
+      CVAT_HOST: cvat.evocargo.com 
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./letsencrypt-webroot:/var/tmp/letsencrypt-webroot
+      - /etc/ssl/private:/etc/ssl/private
+
+  cvat:
+    environment:
+      ALLOWED_HOSTS: '*'
+      CVAT_SHARE_URL: 'Mounted from /home/dkorshunov/cvat/share host directory'
+    volumes:
+      - cvat_share:/home/django/share:ro
+
+volumes:
+  cvat_share:
+    driver_opts:
+      type: none
+      device: /home/evocargo/cvat/share
+      o: bind

installation_quide.md

@@ -0,0 +1,169 @@
+# Server init setup
+
+[Default server setup](https://www.digitalocean.com/community/tutorials/initial-server-setup-with-ubuntu-16-04#step-five-%E2%80%94-disable-password-authentication-(recommended))
+
+# CVAT setup
+
+## 1. Install docker
+
+```bash
+sudo apt-get update
+sudo apt-get --no-install-recommends install -y \
+  apt-transport-https \
+  ca-certificates \
+  curl \
+  gnupg-agent \
+  software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository \
+  "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) \
+  stable"
+sudo apt-get update
+sudo apt-get --no-install-recommends install -y docker-ce docker-ce-cli containerd.io
+```
+
+## 2. Perform post-installation steps to run docker without root permissions
+
+sudo groupadd docker
+sudo usermod -aG docker $USER
+
+### Rebooting...
+
+## 3. Install docker-compose
+
+sudo apt-get --no-install-recommends install -y python3-pip python3-setuptools
+sudo python3 -m pip install setuptools docker-compose
+
+## 4. Clone CVAT source code from the GitHub repository.
+
+sudo apt-get --no-install-recommends install -y git
+git clone [email protected]:EvoCargo/cvat.git
+cd cvat
+git checkout evocargo
+
+
+## 5. Create docker-compose.override.yml 
+
+
+NUCLIO_VERSION=1.6.1
+cat <<EOT >> docker-compose.override.yml
+version: '3.3'
+
+services:
+  cvat_proxy:
+    environment:
+      CVAT_HOST: 'cvat.evocargo.com'
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./letsencrypt-webroot:/var/tmp/letsencrypt-webroot
+      - /etc/ssl/private:/etc/ssl/private
+
+  serverless:
+    container_name: nuclio
+    image: quay.io/nuclio/dashboard:1.6.1-amd64
+
+  cvat:
+    environment:
+      ALLOWED_HOSTS: '*'
+      CVAT_SHARE_URL: 'Mounted from /home/dkorshunov/cvat/share host directory'
+    volumes:
+      - cvat_share:/home/django/share:ro
+    build:
+      context: .
+      args:
+        http_proxy:
+        https_proxy:
+        no_proxy: nuclio,${no_proxy}
+        socks_proxy:
+        USER: 'django'
+        DJANGO_CONFIGURATION: 'production'
+        TZ: 'Etc/UTC'
+        CLAM_AV: 'no'  vim 
+
+volumes:
+  cvat_share:
+    driver_opts:
+      type: none
+      device: /home/evocargo/cvat/share
+      o: bind
+
+EOT
+
+
+## 6. Run docker containers.
+
+docker-compose \
+  -f docker-compose.yml \
+  -f docker-compose.dev.yml \
+  -f docker-compose.override.yml \
+  -f components/analytics/docker-compose.analytics.yml \
+  -f components/serverless/docker-compose.serverless.yml \
+  up -d --build
+
+
+## 7. Create superuser
+
+docker exec -it cvat bash -ic 'python3 ~/manage.py createsuperuser'
+
+## 8. Deploy secure CVAT instance with HTTPS
+
+[Follow the guide](https://github.com/openvinotoolkit/cvat/blob/develop/cvat/apps/documentation/installation.md#deploy-secure-cvat-instance-with-https)
+
+
+## 9. To install nuctl tool to build and deploy serverless functions.
+
+```
+NUCLIO_VERSION=1.6.1
+wget https://github.com/nuclio/nuclio/releases/download/$NUCLIO_VERSION/nuctl-$NUCLIO_VERSION-linux-amd64
+sudo chmod +x nuctl-$NUCLIO_VERSION-linux-amd64
+sudo ln -sf $(pwd)/nuctl-$NUCLIO_VERSION-linux-amd64 /usr/local/bin/nuctl
+nuctl create project cvat
+
+./serverless/deploy_cpu.sh
+./serverless/deploy_gpu.sh
+```
+
+
+# Backup data
+
+docker-compose stop
+
+mkdir backup
+docker run --rm --name temp_backup --volumes-from cvat_db -v $(pwd)/backup:/backup ubuntu tar -cjvf /backup/cvat_db.tar.bz2 /var/lib/postgresql/data
+docker run --rm --name temp_backup --volumes-from cvat -v $(pwd)/backup:/backup ubuntu tar -cjvf /backup/cvat_data.tar.bz2 /home/django/data
+docker run --rm --name temp_backup --volumes-from cvat_elasticsearch -v $(pwd)/backup:/backup ubuntu tar -cjvf /backup/cvat_events.tar.bz2 /usr/share/elasticsearch/data
+
+
+docker-compose \
+  -f docker-compose.yml \
+  -f docker-compose.dev.yml \
+  -f docker-compose.override.yml \
+  -f components/analytics/docker-compose.analytics.yml \
+  -f components/serverless/docker-compose.serverless.yml \
+  up -d --build
+
+
+#Restor data
+
+docker-compose stop
+
+cd backup
+
+docker run --rm --name temp_backup --volumes-from cvat_db -v $(pwd):/backup ubuntu bash -c "cd /var/lib/postgresql/data && tar -xvf /backup/cvat_db.tar.bz2 --strip 4"
+docker run --rm --name temp_backup --volumes-from cvat -v $(pwd):/backup ubuntu bash -c "cd /home/django/data && tar -xvf /backup/cvat_data.tar.bz2 --strip 3"
+docker run --rm --name temp_backup --volumes-from cvat_elasticsearch -v $(pwd):/backup ubuntu bash -c "cd /usr/share/elasticsearch/data && tar -xvf /backup/cvat_events.tar.bz2 --strip 4"
+
+cd ..
+
+docker-compose \
+  -f docker-compose.yml \
+  -f docker-compose.dev.yml \
+  -f docker-compose.override.yml \
+  -f components/analytics/docker-compose.analytics.yml \
+  -f components/serverless/docker-compose.serverless.yml \
+  up -d --build
+
+