Add dedicated Workers and Webhooks Instances (#24)

* Add dedicated Workers and Webhooks Instances * disable scaling per default enable redis persistence when used * Delete commands.shj * ignore debug yaml files * change pruneData default cleanup values.yaml from basicAuth * increase chart version * update redis dependency * ignore all custom values * remote test annotations * move deploymentPodEnvironments into helper * add worker and webhook resource limits * extend readme * increase n8n version to 0.196.0 Co-authored-by: Stefan Warnat <[email protected]>
8gears · Jan 8, 2023 · 466e302 · 466e302
1 parent 90093a1
commit 466e302
Show file tree

Hide file tree

Showing 13 changed files with 390 additions and 19 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+/values.*.yaml
+commands.sh
+yamls
diff --git a/Chart.lock b/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 17.2.0
+digest: sha256:989a8e3f05d54b04db15e782bd24ca5a5872a36f5dc02b7882b25491a5a5ddde
+generated: "2022-09-30T08:49:11.0457529+02:00"
diff --git a/Chart.yaml b/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: n8n
-version: 0.7.0
-appVersion: 0.209.4
+version: 0.8.0
+appVersion: 0.210.0
 type: application
 
 description: A Kubernetes Helm chart for n8n a free and open fair-code licensed node based Workflow Automation Tool. Easily automate tasks across different services.
@@ -19,3 +19,15 @@ sources:
   - https://n8n.io/
 maintainers:
   - name: Vadim Bauer
+
+dependencies:
+  - name: redis
+    version: 17.2.0
+    repository: https://charts.bitnami.com/bitnami
+    condition: redis.enabled
+
+dependencies:
+  - name: redis
+    version: 17.2.0
+    repository: https://charts.bitnami.com/bitnami
+    condition: redis.enabled
diff --git a/README.md b/README.md
@@ -225,6 +225,25 @@ nodeSelector: { }
 tolerations: [ ]
 
 affinity: { }
+
+scaling:
+  enabled: false
+
+  worker:
+    count: 2
+    concurrency: 2
+
+  webhook:
+    enabled: false
+    count: 1
+
+  redis:
+    host: 
+    password:
+
+redis:
+  enabled: false
+  # Other default redis values: https://github.com/bitnami/charts/blob/master/bitnami/redis/values.yaml
 ```
 
 # Typical Values Example
@@ -245,6 +264,41 @@ secret:
       password: 'big secret'
 
 ```
+## Setup
+
+```shell
+helm install -f values.yaml -n n8n deploymentname n8n
+```
+
+## Scaling
+
+n8n provides a **queue-mode**, where the workload is shared between multiple instances of same n8n installation.   
+This provide a shared load over multiple instances and a limited high availability, because the controller instance remain as Single-Point-Of-Failure.  
+
+With the help of an internal/external redis server and by using the excelent BullMQ, the tasks can be shared over different instances, which also can run on different hosts.
+
+[See docs about this Queue-Mode](https://docs.n8n.io/hosting/scaling/queue-mode/)
+
+To enable this mode within this helm chart, you simple should set scaling.enable to true. This chart is configured to spawn by default 2 worker instances.
+
+```yaml
+scaling:
+  enabled: true
+```
+
+You can define to spawn more worker, by set scaling.worker.count to a higher number. Also it is possible to define your own external redis server.
+
+```yaml
+scaling:
+  enabled: true
+  redis:
+    host: "redis-hostname"
+    password: "redis-password-if-set"
+```
+
+If you want to use the internal redis server, set **redis.enable** to "**true**". By default no redis server is spawned.
+
+At last scaling option is it possible to create dedicated webhook instances, which only process the webhooks. If you set **scaling.webhook.enabled** to "true", then webhook processing on main instance is disabled and by default a single webhook instance is started.
 
 ## Chart Deployment
 
@@ -255,3 +309,4 @@ helm repo add  --username='robot$helmcli' --password="$PASSWD" open-8gears https
 helm push --username='robot$helmcli' --password="$PASSWD" . open-8gears
 
 ```
+
diff --git a/charts/redis-17.2.0.tgz b/charts/redis-17.2.0.tgz
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
@@ -50,6 +50,45 @@ app.kubernetes.io/name: {{ include "n8n.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{/*
+Selector labels
+*/}}
+{{- define "n8n.deploymentPodEnvironments" -}}
+{{- range $key, $value := .Values.extraEnv }}
+- name: {{ $key }}
+  value: {{ $value | quote}}
+{{ end }}
+- name: "N8N_PORT" #! we better set the port once again as ENV Var, see: https://community.n8n.io/t/default-config-is-not-set-or-the-port-to-be-more-precise/3158/3?u=vad1mo
+  value: {{ get .Values.config "port" | default "5678" | quote }}
+- name: "N8N_ENCRYPTION_KEY"
+  valueFrom:
+    secretKeyRef:
+      key:  N8N_ENCRYPTION_KEY
+      name: {{ include "n8n.fullname" . }}
+{{- if or .Values.config .Values.secret }}
+- name: "N8N_CONFIG_FILES"
+  value: {{ include "n8n.configFiles" . | quote }}
+{{ end }}
+{{- if .Values.scaling.enabled }}
+- name: "QUEUE_BULL_REDIS_HOST"
+  {{- if .Values.scaling.redis.host }}
+  value: "{{ .Values.scaling.redis.host }}"
+  {{ else }}
+  value: "{{ .Release.Name }}-redis-master"
+  {{ end }}  
+- name: "EXECUTIONS_MODE"
+  value: "queue"
+{{ end }}
+{{- if .Values.scaling.redis.password }}
+- name: "QUEUE_BULL_REDIS_PASSWORD"
+  value: "{{ .Values.scaling.redis.password }}"
+{{ end }}
+{{- if .Values.scaling.webhook.enabled }}
+- name: "N8N_DISABLE_PRODUCTION_MAIN_PROCESS"
+  value: "true"
+{{ end }}
+{{- end }}
+
 {{/*
 Create the name of the service account to use
 */}}

diff --git a/templates/deployment.webhooks.yaml b/templates/deployment.webhooks.yaml
@@ -0,0 +1,95 @@
+{{- if .Values.scaling.webhook.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "n8n.fullname" . }}-webhook
+  labels:
+    {{- include "n8n.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.scaling.webhook.count }}
+  {{- end }}
+  strategy:
+    type: {{ .Values.deploymentStrategy.type }}
+  selector:
+    matchLabels:
+      {{- include "n8n.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/type: webhooks
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ print .Values | sha256sum }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "n8n.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/type: webhooks
+        {{- if .Values.podLabels }}
+          {{ toYaml .Values.podLabels | nindent 8 }}
+        {{- end }}
+
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "n8n.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["n8n"]
+          args: ["webhook"]
+          env:
+            {{- include "n8n.deploymentPodEnvironments" . | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: {{ get .Values.config "port" | default 5678 }}
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.webhookResources | nindent 12 }}
+          volumeMounts:
+            - name: data
+              mountPath: /root/.n8n
+            {{- if .Values.config }}
+            - name: config-volume
+              mountPath: /n8n-config
+            {{- end }}
+            {{- if .Values.secret }}
+            - name: secret-volume
+              mountPath: /n8n-secret
+                {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: "data"
+          {{ include "n8n.pvc" . }}
+        {{- if .Values.config }}
+        - name: config-volume
+          configMap:
+            name: {{ include "n8n.fullname" . }}
+        {{- end }}
+        {{- if .Values.secret }}
+        - name: secret-volume
+          secret:
+            secretName: {{ include "n8n.fullname" . }}
+            items:
+              - key: "secret.json"
+                path: "secret.json"
+        {{- end }}
+{{- end }}
diff --git a/templates/deployment.worker.yaml b/templates/deployment.worker.yaml
@@ -0,0 +1,95 @@
+{{- if .Values.scaling.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "n8n.fullname" . }}-worker
+  labels:
+    {{- include "n8n.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.scaling.worker.count }}
+  {{- end }}
+  strategy:
+    type: {{ .Values.deploymentStrategy.type }}
+  selector:
+    matchLabels:
+      {{- include "n8n.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/type: worker
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ print .Values | sha256sum }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels:
+        {{- include "n8n.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/type: worker
+        {{- if .Values.podLabels }}
+          {{ toYaml .Values.podLabels | nindent 8 }}
+        {{- end }}
+
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "n8n.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["n8n"]
+          args: ["worker", "--concurrency={{ .Values.scaling.worker.concurrency }}"]
+          env:
+            {{- include "n8n.deploymentPodEnvironments" . | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: {{ get .Values.config "port" | default 5678 }}
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.workerResources | nindent 12 }}
+          volumeMounts:
+            - name: data
+              mountPath: /root/.n8n
+            {{- if .Values.config }}
+            - name: config-volume
+              mountPath: /n8n-config
+            {{- end }}
+            {{- if .Values.secret }}
+            - name: secret-volume
+              mountPath: /n8n-secret
+                {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: "data"
+          {{ include "n8n.pvc" . }}
+        {{- if .Values.config }}
+        - name: config-volume
+          configMap:
+            name: {{ include "n8n.fullname" . }}
+        {{- end }}
+        {{- if .Values.secret }}
+        - name: secret-volume
+          secret:
+            secretName: {{ include "n8n.fullname" . }}
+            items:
+              - key: "secret.json"
+                path: "secret.json"
+        {{- end }}
+  {{- end }}
diff --git a/templates/deployment.yaml b/templates/deployment.yaml
@@ -13,6 +13,7 @@ spec:
   selector:
     matchLabels:
       {{- include "n8n.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/type: master
   template:
     metadata:
       annotations:
@@ -22,6 +23,11 @@ spec:
         {{- end }}
       labels:
         {{- include "n8n.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/type: master
+        {{- if .Values.podLabels }}
+          {{ toYaml .Values.podLabels | nindent 8 }}
+        {{- end }}
+
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -37,21 +43,7 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
-            {{- range $key, $value := .Values.extraEnv }}
-            - name: {{ $key }}
-              value: {{ $value | quote}}
-            {{ end }}
-            - name: "N8N_PORT" #! we better set the port once again as ENV Var, see: https://community.n8n.io/t/default-config-is-not-set-or-the-port-to-be-more-precise/3158/3?u=vad1mo
-              value: {{ get .Values.config "port" | default "5678" | quote }}
-            - name: "N8N_ENCRYPTION_KEY"
-              valueFrom:
-                secretKeyRef:
-                  key:  N8N_ENCRYPTION_KEY
-                  name: {{ include "n8n.fullname" . }}
-            {{- if or .Values.config .Values.secret }}
-            - name: "N8N_CONFIG_FILES"
-              value: {{ include "n8n.configFiles" . | quote }}
-            {{ end }}
+            {{- include "n8n.deploymentPodEnvironments" . | nindent 12 }}
           ports:
             - name: http
               containerPort: {{ get .Values.config "port" | default 5678 }}