Add agent readreplica resources

Makefile.d/k8s.mk

@@ -86,6 +86,7 @@ k8s/vald/deploy:
 	@echo "Permitting error because there's some cases nothing to apply"
 	kubectl apply -f $(TEMP_DIR)/vald/templates/manager/index || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/agent || true
+	kubectl apply -f $(TEMP_DIR)/vald/templates/agent/readreplica || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/discoverer || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/lb || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/correction || true
@@ -116,6 +117,7 @@ k8s/vald/delete:
 	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb
 	kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index
 	kubectl delete -f $(TEMP_DIR)/vald/templates/discoverer
+	kubectl delete -f $(TEMP_DIR)/vald/templates/agent/readreplica || true
 	kubectl delete -f $(TEMP_DIR)/vald/templates/agent
 	rm -rf $(TEMP_DIR)
 

charts/vald-helm-operator/crds/valdrelease.yaml

@@ -385,6 +385,27 @@ spec:
                       x-kubernetes-preserve-unknown-fields: true
                     progressDeadlineSeconds:
                       type: integer
+                    readreplica:
+                      type: object
+                      properties:
+                        enabled:
+                          type: boolean
+                        label_key:
+                          type: string
+                        name:
+                          type: string
+                        replica:
+                          type: integer
+                        service:
+                          type: object
+                          properties:
+                            annotations:
+                              type: object
+                              x-kubernetes-preserve-unknown-fields: true
+                        snapshot_classname:
+                          type: string
+                        volume_name:
+                          type: string
                     resources:
                       type: object
                       properties:
@@ -8647,8 +8668,14 @@ spec:
                                       properties:
                                         enabled:
                                           type: boolean
+                                podSecurityContext:
+                                  type: object
+                                  x-kubernetes-preserve-unknown-fields: true
                                 read_replica_id:
                                   type: string
+                                securityContext:
+                                  type: object
+                                  x-kubernetes-preserve-unknown-fields: true
                                 server_config:
                                   type: object
                                   properties:

charts/vald/templates/agent/readreplica/deployment.yaml

@@ -0,0 +1,155 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- $agent := .Values.agent -}}
+{{- $readreplica := .Values.agent.readreplica -}}
+{{- $defaults := .Values.defaults -}}
+{{- $release := .Release -}}
+{{- $chart := .Chart -}}
+{{- $valdname := include "vald.name" . -}}
+{{- $valdchart := include "vald.chart" . -}}
+# if agent.enabled is true and agent.kind is StatefulSet and readreplica.enabled is true
+# then create StatefulSet for agent and readreplica
+{{- if and $agent.enabled (eq .Values.agent.kind "StatefulSet") $readreplica.enabled }}
+{{ range $id := until (int $agent.minReplicas) }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $readreplica.name }}-{{ $id }}
+  labels:
+    app: {{ $readreplica.name }}-{{ $id }}
+    app.kubernetes.io/name: {{ $valdname }}
+    helm.sh/chart: {{ $valdchart }}
+    app.kubernetes.io/managed-by: {{ $release.Service }}
+    app.kubernetes.io/instance: {{ $release.Name }}
+    app.kubernetes.io/version: {{ $chart.Version }}
+    app.kubernetes.io/component: {{ $readreplica.component_name }}
+    {{ $readreplica.label_key }}: "{{ $id }}"
+  {{- if $agent.annotations }}
+  annotations:
+    {{- toYaml $agent.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if not $agent.hpa.enabled }}
+  replicas: {{ $readreplica.replica }}
+  {{- end }}
+  revisionHistoryLimit: {{ $agent.revisionHistoryLimit }}
+  selector:
+    matchLabels:
+      app: {{ $readreplica.name }}-{{ $id }}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: {{ $readreplica.name }}-{{ $id }}
+        app.kubernetes.io/name: {{ $valdname }}
+        app.kubernetes.io/instance: {{ $release.Name }}
+        app.kubernetes.io/component: {{ $readreplica.component_name }}
+      {{- $pprof := default $defaults.server_config.metrics.pprof $agent.server_config.metrics.pprof }}
+    spec:
+      {{- if $agent.initContainers }}
+      initContainers:
+        {{- if $agent.initContainers }}
+        {{- $initContainers := dict "initContainers" $agent.initContainers "Values" .Values "namespace" .Release.Namespace -}}
+        {{- include "vald.initContainers" $initContainers | trim | nindent 8 }}
+        {{- end }}
+      {{- end }}
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                    - {{ $readreplica.name }}-{{ $id }}
+                topologyKey: kubernetes.io/hostname
+              weight: 100
+      {{- if $agent.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml $agent.topologySpreadConstraints | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ $agent.name }}
+          image: "{{ $agent.image.repository }}:{{ default $defaults.image.tag $agent.image.tag }}"
+          imagePullPolicy: {{ $agent.image.pullPolicy }}
+          {{- $servers := dict "Values" $agent.server_config "default" $defaults.server_config -}}
+          {{- include "vald.containerPorts" $servers | trim | nindent 10 }}
+          resources:
+            {{- toYaml $agent.resources | nindent 12 }}
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          {{- if $agent.securityContext }}
+          securityContext:
+            {{- toYaml $agent.securityContext | nindent 12 }}
+          {{- end }}
+          {{- if $agent.env }}
+          env:
+            {{- toYaml $agent.env | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: {{ $agent.name }}-config
+              mountPath: /etc/server/
+            {{- if not $agent.ngt.enable_in_memory_mode }}
+            {{- if $agent.ngt.index_path }}
+            {{- if $agent.persistentVolume.enabled }}
+            - name: {{ $readreplica.volume_name }}
+              mountPath: {{ dir $agent.ngt.index_path }}
+              mountPropagation: {{ $agent.persistentVolume.mountPropagation }}
+            {{- else }}
+            - name: {{ $agent.name }}-local
+              mountPath: {{ dir $agent.ngt.index_path }}
+            {{- end }}
+            {{- end }}
+            {{- end }}
+            {{- if $agent.volumeMounts }}
+            {{- toYaml $agent.volumeMounts | nindent 12 }}
+            {{- end }}
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      {{- if $agent.podSecurityContext }}
+      securityContext:
+        {{- toYaml $agent.podSecurityContext | nindent 8 }}
+      {{- end }}
+      terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }}
+      volumes:
+        - name: {{ $agent.name }}-config
+          configMap:
+            defaultMode: 420
+            name: {{ $agent.name }}-config
+        - name: {{ $readreplica.volume_name }}
+          persistentVolumeClaim:
+            claimName: {{ $readreplica.name }}-pvc-{{ $id }}
+      {{- if $agent.nodeName }}
+      nodeName: {{ $agent.nodeName }}
+      {{- end }}
+      {{- if $agent.nodeSelector }}
+      nodeSelector:
+        {{- toYaml $agent.nodeSelector | nindent 8 }}
+      {{- end }}
+      {{- if $agent.tolerations }}
+      tolerations:
+        {{- toYaml $agent.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if $agent.podPriority }}
+      {{- if $agent.podPriority.enabled }}
+      priorityClassName: {{ $release.Namespace }}-{{ $agent.name }}-priority
+      {{- end }}
+      {{- end }}
+{{- end }}
+{{- end }}

charts/vald/templates/agent/readreplica/networkpolicy.yaml

@@ -0,0 +1,79 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{{- $agent       := .Values.agent -}}
+{{- $readreplica := .Values.agent.readreplica -}}
+{{- $lb          := .Values.gateway.lb -}}
+{{- $index       := .Values.manager.index -}}
+{{- $corrector   := .Values.manager.index.corrector -}}
+{{- $creator     := .Values.manager.index.creator -}}
+{{- $saver       := .Values.manager.index.saver -}}
+{{- if .Values.defaults.networkPolicy.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: agent-readreplica-allow
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/component: {{ $readreplica.component_name }}
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
+          podSelector:
+            matchLabels:
+              app: {{ $lb.name }}
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
+          podSelector:
+            matchLabels:
+              app: {{ $index.name }}
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
+          podSelector:
+            matchLabels:
+              app: {{ $corrector.name }}
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
+          podSelector:
+            matchLabels:
+              app: {{ $creator.name }}
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
+          podSelector:
+            matchLabels:
+              app: {{ $saver.name }}
+    {{- if .Values.defaults.networkPolicy.custom.ingress }}
+    {{- toYaml .Values.defaults.networkPolicy.custom.ingress | nindent 4 }}
+    {{- end }}
+  egress:
+    - to:
+      - namespaceSelector:
+          matchLabels:
+            kubernetes.io/metadata.name: kube-system
+    {{- if .Values.defaults.networkPolicy.custom.egress }}
+    {{- toYaml .Values.defaults.networkPolicy.custom.egress | nindent 4 }}
+    {{- end }}
+{{- end }}

charts/vald/templates/agent/readreplica/pvc.yaml

@@ -0,0 +1,39 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{{- $agent := .Values.agent -}}
+{{- $readreplica := .Values.agent.readreplica -}}
+{{- if $readreplica.enabled }}
+{{ range $id := until (int $agent.minReplicas) }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ $readreplica.name }}-pvc-{{ $id }}
+  labels:
+    {{ $readreplica.label_key }}: "{{ $id }}"
+spec:
+  accessModes:
+    - ReadOnlyMany
+  resources:
+    requests:
+      storage: 1Gi
+  dataSource:
+    name: {{ $readreplica.name }}-snapshot-{{ $id }}
+    kind: VolumeSnapshot
+    apiGroup: snapshot.storage.k8s.io
+{{- end }}
+{{- end }}

charts/vald/templates/agent/readreplica/snapshot.yaml

@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- $agent := .Values.agent -}}
+{{- $readreplica := .Values.agent.readreplica -}}
+{{- if $readreplica.enabled }}
+{{ range $id := until (int $agent.minReplicas) }}
+---
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  labels:
+    {{ $readreplica.label_key }}: "{{ $id }}"
+  name: {{ $readreplica.name }}-snapshot-{{ $id }}
+spec:
+  volumeSnapshotClassName: csi-hostpath-snapclass
+  source:
+    # Specify the PVCs for which you want to take snapshots
+    persistentVolumeClaimName: {{ $agent.name }}-pvc-{{ $agent.name }}-{{ $id }}
+{{- end }}
+{{- end }}