From 6c9ee2fc85981eab928c76a5e701e44f8390038d Mon Sep 17 00:00:00 2001
From: Roman Donchenko <roman@cvat.ai>
Date: Mon, 4 Sep 2023 12:23:34 +0300
Subject: [PATCH] helm-chart: make sure database migrations are only run by one
 process (#6780)

To do that, remove the running of migrations from the server pod(s), and
add a Kubernetes job to do it instead.

This will prevent Django from corrupting the database when two or more
instances are started up at the same time.
---
 CHANGELOG.md                                  |  4 +-
 helm-chart/Chart.yaml                         |  2 +-
 .../cvat_backend/initializer/job.yml          | 72 +++++++++++++++++++
 .../cvat_backend/server/deployment.yml        |  2 +-
 helm-chart/values.yaml                        |  9 +++
 5 files changed, 86 insertions(+), 3 deletions(-)
 create mode 100644 helm-chart/templates/cvat_backend/initializer/job.yml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2acbc678de40..7b9d809cb9ce 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - TDB
 
 ### Changed
-- TDB
+- \[Helm\] Database migrations now run in a separate job instead of the server pod,
+  in order to avoid data corruption when multiple replicas of the server are used
+  (<https://github.com/opencv/cvat/pull/6780>)
 
 ### Deprecated
 - TDB
diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml
index 959369cee08f..232181a7ec45 100644
--- a/helm-chart/Chart.yaml
+++ b/helm-chart/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.1
+version: 0.10.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
diff --git a/helm-chart/templates/cvat_backend/initializer/job.yml b/helm-chart/templates/cvat_backend/initializer/job.yml
new file mode 100644
index 000000000000..149779e7c31a
--- /dev/null
+++ b/helm-chart/templates/cvat_backend/initializer/job.yml
@@ -0,0 +1,72 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  # Jobs are mostly immutable, so Helm can't update them when the user upgrades
+  # a release. Work around this by including the revision in the job name, so that
+  # every upgrade creates a new job.
+  # See also <https://github.com/helm/helm/issues/7082>.
+  name: {{ .Release.Name }}-backend-initializer-r{{ .Release.Revision }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: cvat-app
+    tier: backend
+    component: initializer
+    {{- include "cvat.labels" . | nindent 4 }}
+    {{- with .Values.cvat.backend.initializer.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.cvat.backend.initializer.annotations }}
+  annotations:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  template:
+    metadata:
+      labels:
+        app: cvat-app
+        tier: backend
+        component: initializer
+        {{- include "cvat.labels" . | nindent 8 }}
+        {{- with .Values.cvat.backend.initializer.labels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- with .Values.cvat.backend.initializer.annotations }}
+      annotations:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      containers:
+        - name: cvat-app-backend-initializer-container
+          image: {{ .Values.cvat.backend.image }}:{{ .Values.cvat.backend.tag }}
+          imagePullPolicy: {{ .Values.cvat.backend.imagePullPolicy }}
+          {{- with .Values.cvat.backend.initializer.resources }}
+          resources:
+          {{- toYaml . | nindent 12 }}
+          {{- end }}
+          args: ["init"]
+          env:
+          {{ include "cvat.sharedBackendEnv" . | indent 10 }}
+          {{- with .Values.cvat.backend.initializer.additionalEnv }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+          {{- with .Values.cvat.backend.initializer.additionalVolumeMounts }}
+          volumeMounts:
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+      restartPolicy: OnFailure
+      {{- with .Values.cvat.backend.initializer.affinity }}
+      affinity:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.cvat.backend.initializer.tolerations }}
+      tolerations:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.cvat.backend.initializer.additionalVolumes }}
+      volumes:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
diff --git a/helm-chart/templates/cvat_backend/server/deployment.yml b/helm-chart/templates/cvat_backend/server/deployment.yml
index 4a79c2cce917..4da797ed2329 100644
--- a/helm-chart/templates/cvat_backend/server/deployment.yml
+++ b/helm-chart/templates/cvat_backend/server/deployment.yml
@@ -51,7 +51,7 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["init", "run", "server"]
+          args: ["run", "server"]
           env:
           - name: ALLOWED_HOSTS
             value: {{ .Values.cvat.backend.server.envs.ALLOWED_HOSTS | squote}}
diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml
index 9eb49a229b8a..ec1ca8f9e7e2 100644
--- a/helm-chart/values.yaml
+++ b/helm-chart/values.yaml
@@ -9,6 +9,15 @@ fullnameOverride: ""
 
 cvat:
   backend:
+    initializer:
+      labels: {}
+      annotations: {}
+      resources: {}
+      affinity: {}
+      tolerations: []
+      additionalEnv: []
+      additionalVolumes: []
+      additionalVolumeMounts: []
     server:
       replicas: 1
       labels: {}