-
Notifications
You must be signed in to change notification settings - Fork 77
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add agent readreplica resources (#2258)
* Implement sample read replica rotation logic * Add external-snapshotter in go.mod * Add readreplica initial implementation * Add service implementation of readreplica rotator * Add readreplica rotate config * Refactor to read labels * replace id to _MY_TARGET_REPLICA_ID_ * Add snapshot k8s client * Format * Fix snapshot client initialization * Add Apache License to main.go * Use GetConfigOrDie * Use internal/k8s client * Refactor * Use controller-runtime for CRUD in readreplica * Remove snapshot client and use controller-runtime * Move LabelSelector into internal/k8s * Report error to span * Update go modules * Remove old example * Refactor * Disable exhaustruct for now * Fix predeclared * Fix stylecheck * nolint:gomnd * Add test template * Add test for getNewBaseName * Add readreplica-rotate build scripts * Add service account and clusterroles for read replica rotator * Update charts * Update comment * Add readreplica deployment templates * Add readreplica deployment template * Add svc for readreplica * Update valdrelease * Add podAntiAffinity for readreplica * Add security context for readreplica * Update valdrelease * Update valdrelease scheme * Fix component name of readreplica so that svc can find it well * Change readreplica svc component name * Change headless to normal ClusterIP to do loadbalancing for readreplica * Fix comment * Parametarize some field in configmap * remove sidecar related things from readreplica deployment * Fix readreplica svc type to ClusterIP * Update README.md * Add networkpolicy * Update docs * Update network policy selector label * Apply format
- Loading branch information
Showing
11 changed files
with
1,260 additions
and
793 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
155 changes: 155 additions & 0 deletions
155
charts/vald/templates/agent/readreplica/deployment.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| # | ||
| # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]> | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # You may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # https://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
| {{- $agent := .Values.agent -}} | ||
| {{- $readreplica := .Values.agent.readreplica -}} | ||
| {{- $defaults := .Values.defaults -}} | ||
| {{- $release := .Release -}} | ||
| {{- $chart := .Chart -}} | ||
| {{- $valdname := include "vald.name" . -}} | ||
| {{- $valdchart := include "vald.chart" . -}} | ||
| # if agent.enabled is true and agent.kind is StatefulSet and readreplica.enabled is true | ||
| # then create StatefulSet for agent and readreplica | ||
| {{- if and $agent.enabled (eq .Values.agent.kind "StatefulSet") $readreplica.enabled }} | ||
| {{ range $id := until (int $agent.minReplicas) }} | ||
| --- | ||
| apiVersion: apps/v1 | ||
| kind: Deployment | ||
| metadata: | ||
| name: {{ $readreplica.name }}-{{ $id }} | ||
| labels: | ||
| app: {{ $readreplica.name }}-{{ $id }} | ||
| app.kubernetes.io/name: {{ $valdname }} | ||
| helm.sh/chart: {{ $valdchart }} | ||
| app.kubernetes.io/managed-by: {{ $release.Service }} | ||
| app.kubernetes.io/instance: {{ $release.Name }} | ||
| app.kubernetes.io/version: {{ $chart.Version }} | ||
| app.kubernetes.io/component: {{ $readreplica.component_name }} | ||
| {{ $readreplica.label_key }}: "{{ $id }}" | ||
| {{- if $agent.annotations }} | ||
| annotations: | ||
| {{- toYaml $agent.annotations | nindent 4 }} | ||
| {{- end }} | ||
| spec: | ||
| {{- if not $agent.hpa.enabled }} | ||
| replicas: {{ $readreplica.replica }} | ||
| {{- end }} | ||
| revisionHistoryLimit: {{ $agent.revisionHistoryLimit }} | ||
| selector: | ||
| matchLabels: | ||
| app: {{ $readreplica.name }}-{{ $id }} | ||
| template: | ||
| metadata: | ||
| creationTimestamp: null | ||
| labels: | ||
| app: {{ $readreplica.name }}-{{ $id }} | ||
| app.kubernetes.io/name: {{ $valdname }} | ||
| app.kubernetes.io/instance: {{ $release.Name }} | ||
| app.kubernetes.io/component: {{ $readreplica.component_name }} | ||
| {{- $pprof := default $defaults.server_config.metrics.pprof $agent.server_config.metrics.pprof }} | ||
| spec: | ||
| {{- if $agent.initContainers }} | ||
| initContainers: | ||
| {{- if $agent.initContainers }} | ||
| {{- $initContainers := dict "initContainers" $agent.initContainers "Values" .Values "namespace" .Release.Namespace -}} | ||
| {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} | ||
| {{- end }} | ||
| {{- end }} | ||
| affinity: | ||
| podAntiAffinity: | ||
| preferredDuringSchedulingIgnoredDuringExecution: | ||
| - podAffinityTerm: | ||
| labelSelector: | ||
| matchExpressions: | ||
| - key: app | ||
| operator: In | ||
| values: | ||
| - {{ $readreplica.name }}-{{ $id }} | ||
| topologyKey: kubernetes.io/hostname | ||
| weight: 100 | ||
| {{- if $agent.topologySpreadConstraints }} | ||
| topologySpreadConstraints: | ||
| {{- toYaml $agent.topologySpreadConstraints | nindent 8 }} | ||
| {{- end }} | ||
| containers: | ||
| - name: {{ $agent.name }} | ||
| image: "{{ $agent.image.repository }}:{{ default $defaults.image.tag $agent.image.tag }}" | ||
| imagePullPolicy: {{ $agent.image.pullPolicy }} | ||
| {{- $servers := dict "Values" $agent.server_config "default" $defaults.server_config -}} | ||
| {{- include "vald.containerPorts" $servers | trim | nindent 10 }} | ||
| resources: | ||
| {{- toYaml $agent.resources | nindent 12 }} | ||
| terminationMessagePath: /dev/termination-log | ||
| terminationMessagePolicy: File | ||
| {{- if $agent.securityContext }} | ||
| securityContext: | ||
| {{- toYaml $agent.securityContext | nindent 12 }} | ||
| {{- end }} | ||
| {{- if $agent.env }} | ||
| env: | ||
| {{- toYaml $agent.env | nindent 12 }} | ||
| {{- end }} | ||
| volumeMounts: | ||
| - name: {{ $agent.name }}-config | ||
| mountPath: /etc/server/ | ||
| {{- if not $agent.ngt.enable_in_memory_mode }} | ||
| {{- if $agent.ngt.index_path }} | ||
| {{- if $agent.persistentVolume.enabled }} | ||
| - name: {{ $readreplica.volume_name }} | ||
| mountPath: {{ dir $agent.ngt.index_path }} | ||
| mountPropagation: {{ $agent.persistentVolume.mountPropagation }} | ||
| {{- else }} | ||
| - name: {{ $agent.name }}-local | ||
| mountPath: {{ dir $agent.ngt.index_path }} | ||
| {{- end }} | ||
| {{- end }} | ||
| {{- end }} | ||
| {{- if $agent.volumeMounts }} | ||
| {{- toYaml $agent.volumeMounts | nindent 12 }} | ||
| {{- end }} | ||
| dnsPolicy: ClusterFirst | ||
| restartPolicy: Always | ||
| schedulerName: default-scheduler | ||
| {{- if $agent.podSecurityContext }} | ||
| securityContext: | ||
| {{- toYaml $agent.podSecurityContext | nindent 8 }} | ||
| {{- end }} | ||
| terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }} | ||
| volumes: | ||
| - name: {{ $agent.name }}-config | ||
| configMap: | ||
| defaultMode: 420 | ||
| name: {{ $agent.name }}-config | ||
| - name: {{ $readreplica.volume_name }} | ||
| persistentVolumeClaim: | ||
| claimName: {{ $readreplica.name }}-pvc-{{ $id }} | ||
| {{- if $agent.nodeName }} | ||
| nodeName: {{ $agent.nodeName }} | ||
| {{- end }} | ||
| {{- if $agent.nodeSelector }} | ||
| nodeSelector: | ||
| {{- toYaml $agent.nodeSelector | nindent 8 }} | ||
| {{- end }} | ||
| {{- if $agent.tolerations }} | ||
| tolerations: | ||
| {{- toYaml $agent.tolerations | nindent 8 }} | ||
| {{- end }} | ||
| {{- if $agent.podPriority }} | ||
| {{- if $agent.podPriority.enabled }} | ||
| priorityClassName: {{ $release.Namespace }}-{{ $agent.name }}-priority | ||
| {{- end }} | ||
| {{- end }} | ||
| {{- end }} | ||
| {{- end }} |
79 changes: 79 additions & 0 deletions
79
charts/vald/templates/agent/readreplica/networkpolicy.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| # | ||
| # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]> | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # You may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # https://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| {{- $agent := .Values.agent -}} | ||
| {{- $readreplica := .Values.agent.readreplica -}} | ||
| {{- $lb := .Values.gateway.lb -}} | ||
| {{- $index := .Values.manager.index -}} | ||
| {{- $corrector := .Values.manager.index.corrector -}} | ||
| {{- $creator := .Values.manager.index.creator -}} | ||
| {{- $saver := .Values.manager.index.saver -}} | ||
| {{- if .Values.defaults.networkPolicy.enabled }} | ||
| apiVersion: networking.k8s.io/v1 | ||
| kind: NetworkPolicy | ||
| metadata: | ||
| name: agent-readreplica-allow | ||
| spec: | ||
| podSelector: | ||
| matchLabels: | ||
| app.kubernetes.io/component: {{ $readreplica.component_name }} | ||
| policyTypes: | ||
| - Ingress | ||
| - Egress | ||
| ingress: | ||
| - from: | ||
| - namespaceSelector: | ||
| matchLabels: | ||
| kubernetes.io/metadata.name: {{ .Release.Namespace }} | ||
| podSelector: | ||
| matchLabels: | ||
| app: {{ $lb.name }} | ||
| - namespaceSelector: | ||
| matchLabels: | ||
| kubernetes.io/metadata.name: {{ .Release.Namespace }} | ||
| podSelector: | ||
| matchLabels: | ||
| app: {{ $index.name }} | ||
| - namespaceSelector: | ||
| matchLabels: | ||
| kubernetes.io/metadata.name: {{ .Release.Namespace }} | ||
| podSelector: | ||
| matchLabels: | ||
| app: {{ $corrector.name }} | ||
| - namespaceSelector: | ||
| matchLabels: | ||
| kubernetes.io/metadata.name: {{ .Release.Namespace }} | ||
| podSelector: | ||
| matchLabels: | ||
| app: {{ $creator.name }} | ||
| - namespaceSelector: | ||
| matchLabels: | ||
| kubernetes.io/metadata.name: {{ .Release.Namespace }} | ||
| podSelector: | ||
| matchLabels: | ||
| app: {{ $saver.name }} | ||
| {{- if .Values.defaults.networkPolicy.custom.ingress }} | ||
| {{- toYaml .Values.defaults.networkPolicy.custom.ingress | nindent 4 }} | ||
| {{- end }} | ||
| egress: | ||
| - to: | ||
| - namespaceSelector: | ||
| matchLabels: | ||
| kubernetes.io/metadata.name: kube-system | ||
| {{- if .Values.defaults.networkPolicy.custom.egress }} | ||
| {{- toYaml .Values.defaults.networkPolicy.custom.egress | nindent 4 }} | ||
| {{- end }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| # | ||
| # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]> | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # You may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # https://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| {{- $agent := .Values.agent -}} | ||
| {{- $readreplica := .Values.agent.readreplica -}} | ||
| {{- if $readreplica.enabled }} | ||
| {{ range $id := until (int $agent.minReplicas) }} | ||
| --- | ||
| apiVersion: v1 | ||
| kind: PersistentVolumeClaim | ||
| metadata: | ||
| name: {{ $readreplica.name }}-pvc-{{ $id }} | ||
| labels: | ||
| {{ $readreplica.label_key }}: "{{ $id }}" | ||
| spec: | ||
| accessModes: | ||
| - ReadOnlyMany | ||
| resources: | ||
| requests: | ||
| storage: 1Gi | ||
| dataSource: | ||
| name: {{ $readreplica.name }}-snapshot-{{ $id }} | ||
| kind: VolumeSnapshot | ||
| apiGroup: snapshot.storage.k8s.io | ||
| {{- end }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| # | ||
| # Copyright (C) 2019-2023 vdaas.org vald team <[email protected]> | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # You may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # https://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
| {{- $agent := .Values.agent -}} | ||
| {{- $readreplica := .Values.agent.readreplica -}} | ||
| {{- if $readreplica.enabled }} | ||
| {{ range $id := until (int $agent.minReplicas) }} | ||
| --- | ||
| apiVersion: snapshot.storage.k8s.io/v1 | ||
| kind: VolumeSnapshot | ||
| metadata: | ||
| labels: | ||
| {{ $readreplica.label_key }}: "{{ $id }}" | ||
| name: {{ $readreplica.name }}-snapshot-{{ $id }} | ||
| spec: | ||
| volumeSnapshotClassName: csi-hostpath-snapclass | ||
| source: | ||
| # Specify the PVCs for which you want to take snapshots | ||
| persistentVolumeClaimName: {{ $agent.name }}-pvc-{{ $agent.name }}-{{ $id }} | ||
| {{- end }} | ||
| {{- end }} |
Oops, something went wrong.