From a29c880b7ec50b15d4f9c86e841315929363978e Mon Sep 17 00:00:00 2001 From: Langhalsdino Date: Sun, 28 Feb 2021 19:17:13 +0100 Subject: [PATCH] addd basic cvat deployment k8s templates --- CHANGELOG.md | 1 + kubernetes-templates/01_namespace.yml | 4 + .../02_cvat_backend_storage.yml | 12 ++ kubernetes-templates/02_database_secrets.yml | 13 ++ kubernetes-templates/02_database_storage.yml | 12 ++ .../03_database_deployment.yml | 61 +++++++ kubernetes-templates/03_redis_deployment.yml | 29 ++++ .../04_cvat_backend_deployment.yml | 96 +++++++++++ .../04_cvat_frontend_deployment.yml | 36 +++++ kubernetes-templates/04_database_service.yml | 18 +++ kubernetes-templates/04_redis_service.yml | 18 +++ .../05_cvat_backend_service.yml | 18 +++ .../05_cvat_frontend_service.yml | 18 +++ .../05_cvat_proxy_configmap.yml | 151 ++++++++++++++++++ .../05_cvat_proxy_deployment.yml | 46 ++++++ .../05_cvat_proxy_service.yml | 18 +++ kubernetes-templates/README.md | 40 +++++ 17 files changed, 591 insertions(+) create mode 100644 kubernetes-templates/01_namespace.yml create mode 100644 kubernetes-templates/02_cvat_backend_storage.yml create mode 100644 kubernetes-templates/02_database_secrets.yml create mode 100644 kubernetes-templates/02_database_storage.yml create mode 100644 kubernetes-templates/03_database_deployment.yml create mode 100644 kubernetes-templates/03_redis_deployment.yml create mode 100644 kubernetes-templates/04_cvat_backend_deployment.yml create mode 100644 kubernetes-templates/04_cvat_frontend_deployment.yml create mode 100644 kubernetes-templates/04_database_service.yml create mode 100644 kubernetes-templates/04_redis_service.yml create mode 100644 kubernetes-templates/05_cvat_backend_service.yml create mode 100644 kubernetes-templates/05_cvat_frontend_service.yml create mode 100644 kubernetes-templates/05_cvat_proxy_configmap.yml create mode 100644 kubernetes-templates/05_cvat_proxy_deployment.yml create mode 100644 kubernetes-templates/05_cvat_proxy_service.yml create mode 100644 kubernetes-templates/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 272042f77e71..7b0273a33f75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub () - Project task subsets () +- Kubernetes templates and guide for their deployment () ### Changed diff --git a/kubernetes-templates/01_namespace.yml b/kubernetes-templates/01_namespace.yml new file mode 100644 index 000000000000..fa0aaa12f714 --- /dev/null +++ b/kubernetes-templates/01_namespace.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cvat diff --git a/kubernetes-templates/02_cvat_backend_storage.yml b/kubernetes-templates/02_cvat_backend_storage.yml new file mode 100644 index 000000000000..6d0f51b35006 --- /dev/null +++ b/kubernetes-templates/02_cvat_backend_storage.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cvat-backend-data + namespace: cvat +spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 20Gi diff --git a/kubernetes-templates/02_database_secrets.yml b/kubernetes-templates/02_database_secrets.yml new file mode 100644 index 000000000000..4c811f007284 --- /dev/null +++ b/kubernetes-templates/02_database_secrets.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: cvat-postgres-secret + namespace: cvat + labels: + app: cvat-app + tier: db +stringData: + POSTGRES_DB: cvat + POSTGRES_USER: root + POSTGRES_PASSWORD: POSTGRES_ADMIN_PW diff --git a/kubernetes-templates/02_database_storage.yml b/kubernetes-templates/02_database_storage.yml new file mode 100644 index 000000000000..c331d85b7429 --- /dev/null +++ b/kubernetes-templates/02_database_storage.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cvat-postgres-data + namespace: cvat +spec: + accessModes: + - ReadWriteOnce + storageClassName: standard + resources: + requests: + storage: 20Gi diff --git a/kubernetes-templates/03_database_deployment.yml b/kubernetes-templates/03_database_deployment.yml new file mode 100644 index 000000000000..84bdf3fd62e3 --- /dev/null +++ b/kubernetes-templates/03_database_deployment.yml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-postgres + namespace: cvat + labels: + app: cvat-app + tier: db +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: db + template: + metadata: + labels: + app: cvat-app + tier: db + spec: + containers: + - name: cvat-postgres + image: postgres:10.3-alpine + imagePullPolicy: "IfNotPresent" + env: + - name: POSTGRES_DB + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_DB + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_PASSWORD + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: + - sh + - -c + - su - postgres -c "pg_isready --host=$POD_IP" + initialDelaySeconds: 15 + timeoutSeconds: 2 + resources: {} + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgredb + subPath: postgres + volumes: + - name: postgredb + persistentVolumeClaim: + claimName: cvat-postgres-data diff --git a/kubernetes-templates/03_redis_deployment.yml b/kubernetes-templates/03_redis_deployment.yml new file mode 100644 index 000000000000..eafb52836d0b --- /dev/null +++ b/kubernetes-templates/03_redis_deployment.yml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-redis + namespace: cvat + labels: + app: cvat-app + tier: redis-app +spec: + replicas: 1 + selector: + matchLabels: + app: cvat-app + tier: redis-app + template: + metadata: + labels: + app: cvat-app + tier: redis-app + spec: + containers: + - image: redis:4.0.5-alpine + name: cvat-redis + imagePullPolicy: Always + ports: + - containerPort: 6379 + resources: + limits: + cpu: "0.1" diff --git a/kubernetes-templates/04_cvat_backend_deployment.yml b/kubernetes-templates/04_cvat_backend_deployment.yml new file mode 100644 index 000000000000..7261f0696c9f --- /dev/null +++ b/kubernetes-templates/04_cvat_backend_deployment.yml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-backend + namespace: cvat + labels: + app: cvat-app + tier: backend +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: backend + template: + metadata: + labels: + app: cvat-app + tier: backend + spec: + containers: + - name: cvat-backend-app-container + image: {my.registry.backend.image:develop} + imagePullPolicy: Always + resources: + requests: + cpu: 10m + memory: 100Mi + env: + - name: DJANGO_MODWSGI_EXTRA_ARGS + value: "" + - name: UI_PORT + value: "80" + - name: UI_HOST + value: "cvat-frontend-service" + - name: ALLOWED_HOSTS + value: "{MY_SERVER_URL_COM}" + - name: CVAT_REDIS_HOST + value: "cvat-redis-service" + - name: CVAT_POSTGRES_HOST + value: "cvat-postgres-service" + - name: CVAT_POSTGRES_USER + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_USER + - name: CVAT_POSTGRES_DBNAME + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_DB + - name: CVAT_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-postgres-secret + key: POSTGRES_PASSWORD + ports: + - containerPort: 8080 + volumeMounts: + - mountPath: /home/django/data + name: cvat-backend-data + subPath: data + - mountPath: /home/django/keys + name: cvat-backend-data + subPath: keys + - mountPath: /home/django/logs + name: cvat-backend-data + subPath: logs + - mountPath: /home/django/models + name: cvat-backend-data + subPath: models + initContainers: + - name: user-data-permission-fix + image: busybox + command: ["/bin/chmod", "-R", "777", "/home/django"] + volumeMounts: + - mountPath: /home/django/data + name: cvat-backend-data + subPath: data + - mountPath: /home/django/keys + name: cvat-backend-data + subPath: keys + - mountPath: /home/django/logs + name: cvat-backend-data + subPath: logs + - mountPath: /home/django/models + name: cvat-backend-data + subPath: models + volumes: + - name: cvat-backend-data + persistentVolumeClaim: + claimName: cvat-backend-data + imagePullSecrets: + - name: gitlab-registry diff --git a/kubernetes-templates/04_cvat_frontend_deployment.yml b/kubernetes-templates/04_cvat_frontend_deployment.yml new file mode 100644 index 000000000000..2f29aa31c815 --- /dev/null +++ b/kubernetes-templates/04_cvat_frontend_deployment.yml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-frontend + namespace: cvat + labels: + app: cvat-app + tier: frontend +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: frontend + template: + metadata: + labels: + app: cvat-app + tier: frontend + spec: + containers: + - name: cvat-frontend-app-container + image: {my.registry.fronend.image:develop} + imagePullPolicy: always + env: + - name: REACT_APP_API_PROTOCOL + value: "http" + - name: REACT_APP_API_HOST + value: "{MY_SERVER_URL_COM}" + - name: REACT_APP_API_PORT + value: "8080" + ports: + - containerPort: 80 + resources: {} diff --git a/kubernetes-templates/04_database_service.yml b/kubernetes-templates/04_database_service.yml new file mode 100644 index 000000000000..bb1d04f4065a --- /dev/null +++ b/kubernetes-templates/04_database_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-postgres-service + namespace: cvat + labels: + app: cvat-app + tier: db +spec: + type: ClusterIP + selector: + app: cvat-app + tier: db + ports: + - port: 5432 + targetPort: 5432 + protocol: TCP + name: http diff --git a/kubernetes-templates/04_redis_service.yml b/kubernetes-templates/04_redis_service.yml new file mode 100644 index 000000000000..436b99b27577 --- /dev/null +++ b/kubernetes-templates/04_redis_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-redis-service + namespace: cvat + labels: + app: cvat-app + tier: redis-app +spec: + type: ClusterIP + selector: + app: cvat-app + tier: redis-app + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP + name: http diff --git a/kubernetes-templates/05_cvat_backend_service.yml b/kubernetes-templates/05_cvat_backend_service.yml new file mode 100644 index 000000000000..d6c4659cb4f9 --- /dev/null +++ b/kubernetes-templates/05_cvat_backend_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-backend-service + namespace: cvat + labels: + app: cvat-app + tier: backend +spec: + type: ClusterIP + selector: + app: cvat-app + tier: backend + ports: + - port: 8080 + targetPort: 8080 + protocol: TCP + name: http diff --git a/kubernetes-templates/05_cvat_frontend_service.yml b/kubernetes-templates/05_cvat_frontend_service.yml new file mode 100644 index 000000000000..0c97278c60e3 --- /dev/null +++ b/kubernetes-templates/05_cvat_frontend_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-frontend-service + namespace: cvat + labels: + app: cvat-app + tier: frontend +spec: + type: ClusterIP + selector: + app: cvat-app + tier: frontend + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http diff --git a/kubernetes-templates/05_cvat_proxy_configmap.yml b/kubernetes-templates/05_cvat_proxy_configmap.yml new file mode 100644 index 000000000000..a7ca810626ee --- /dev/null +++ b/kubernetes-templates/05_cvat_proxy_configmap.yml @@ -0,0 +1,151 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cvat-nginx-conf + namespace: cvat +data: + nginx.conf: | + worker_processes 2; + + error_log /dev/stdout info; + + events { + worker_connections 1024; + } + + http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + # For long domain names (e.g. AWS hosts) + server_names_hash_bucket_size 128; + + include /etc/nginx/cvat.d/*.conf; + client_max_body_size 0; + } + cvat.conf: | + server { + listen 80; + server_name _ default; + return 404; + } + + server { + listen 80; + server_name {MY_SERVER_URL_COM}; + + proxy_pass_header X-CSRFToken; + proxy_set_header Host $http_host; + proxy_pass_header Set-Cookie; + + location ~* /api/.*|git/.*|tensorflow/.*|auto_annotation/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.*|dextr/.*|reid/.* { + proxy_pass http://cvat-backend-service:8080; + } + + # workaround for match location by arguments + location = / { + error_page 418 = @annotation_ui; + + if ( $query_string ~ "^id=\d+.*" ) { return 418; } + proxy_pass http://cvat-frontend-service:80; + } + + location / { + proxy_pass http://cvat-frontend-service:80; + } + + # old annotation ui, will be removed in the future. + location @annotation_ui { + proxy_pass http://cvat-backend-service:8080; + } + } + mime.types: | + types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + image/svg+xml svg svgz; + image/webp webp; + + application/font-woff woff; + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.wap.wmlc wmlc; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; + application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; + } diff --git a/kubernetes-templates/05_cvat_proxy_deployment.yml b/kubernetes-templates/05_cvat_proxy_deployment.yml new file mode 100644 index 000000000000..456bec2ea901 --- /dev/null +++ b/kubernetes-templates/05_cvat_proxy_deployment.yml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cvat-nginx + namespace: cvat + labels: + app: cvat-app + tier: proxy +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: cvat-app + tier: proxy + template: + metadata: + labels: + app: cvat-app + tier: proxy + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /etc/nginx + readOnly: true + name: cvat-nginx-conf + - mountPath: /var/log/nginx + name: log + volumes: + - name: cvat-nginx-conf + configMap: + name: cvat-nginx-conf + items: + - key: nginx.conf + path: nginx.conf + - key: mime.types + path: mime.types + - key: cvat.conf + path: cvat.d/cvat.conf + - name: log + emptyDir: {} diff --git a/kubernetes-templates/05_cvat_proxy_service.yml b/kubernetes-templates/05_cvat_proxy_service.yml new file mode 100644 index 000000000000..18229d3b3dd7 --- /dev/null +++ b/kubernetes-templates/05_cvat_proxy_service.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cvat-proxy-service + namespace: cvat + labels: + app: cvat-app + tier: proxy +spec: + type: NodePort + selector: + app: cvat-app + tier: proxy + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http diff --git a/kubernetes-templates/README.md b/kubernetes-templates/README.md new file mode 100644 index 000000000000..c601892ff0d3 --- /dev/null +++ b/kubernetes-templates/README.md @@ -0,0 +1,40 @@ +# Deploying cvat in a kubernetes cluster + +This guide will focus on how to deploy cvat in an kubernetes environment. It was tested on Kubernetes v1.19.3 but should work for >=v1.9, eventhough it is untested. + +## Building the container +In order to do so, you will first need to build the cvat backend and frontend images and push them to a registry that you can pull from within the cluster +``` +echo "Building backend" +docker build --cache-from $CI_REGISTRY_IMAGE/backend:release-1.1.0 \ + --build-arg TF_ANNOTATION=no --build-arg AUTO_SEGMENTATION=no \ + --build-arg WITH_TESTS=no --build-arg TZ="Etc/UTC" --build-arg OPENVINO_TOOLKIT=no \ + --build-arg USER=django --build-arg DJANGO_CONFIGURATION=production \ + --build-arg TZ="Etc/UTC" . +docker push $CI_REGISTRY_IMAGE/backend:release-1.1.0 + +echo "Building frontend" +docker build --file Dockerfile.ui \ + --tag $CI_REGISTRY_IMAGE/frontend:release-1.1.0 - . +docker push $CI_REGISTRY_IMAGE/frontend:release-1.1.0 +``` + + +## Adjusting the kubernetes templates + +1. Replace the URL pointing to the backend and fronend image in `kubernetes-templates/04_cvat_backend_deployment.yml` and `kubernetes-templates/04_cvat_frontend_deployment.yml`. Furthermore adjusting their pull secrets + +2. Replacing the domain dummy with your real domain name `cvat.my.cool.domain.com`. Replace `{MY_SERVER_URL_COM}` in `kubernetes-templates/04_cvat_frontend_deployment.yml` and `kubernetes-templates/05_cvat_proxy_configmap.yml`. + +3. Insert your choosen database password the `kubernetes-templates/02_database_secrets.yml` + +## Deploying to the cluster +Deploy everything to your cluster with `kubectl apply -f kubernetes-templates/` + +## Create the django super user + +``` +kubectl get pods --namespace cvat +kubectl --namespace cvat exec -it cvat-backend-78c954f84f-qxb8b -- /bin/bash +python3 ~/manage.py createsuperuser +```