Skip to content

Commit

Permalink
Merge branch 'main' into feature/corrector/faster2
Browse files Browse the repository at this point in the history
  • Loading branch information
ykadowak authored Nov 22, 2023
2 parents 2b0715b + 3dc099a commit 1625346
Show file tree
Hide file tree
Showing 28 changed files with 8,694 additions and 589 deletions.
2 changes: 2 additions & 0 deletions Makefile.d/k8s.mk
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ k8s/manifest/update: \
mv $(TEMP_DIR)/vald/templates/gateway/lb k8s/gateway/lb
mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index
mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction
mv $(TEMP_DIR)/vald/templates/index/job/creation k8s/index/job/creation
mv $(TEMP_DIR)/vald/templates/index/job/save k8s/index/job/save
rm -rf $(TEMP_DIR)

.PHONY: k8s/manifest/helm-operator/clean
Expand Down
2,279 changes: 2,114 additions & 165 deletions charts/vald-helm-operator/crds/valdrelease.yaml

Large diffs are not rendered by default.

20 changes: 17 additions & 3 deletions charts/vald/templates/agent/networkpolicy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
# limitations under the License.
#

{{- $agent := .Values.agent -}}
{{- $lb := .Values.gateway.lb -}}
{{- $index := .Values.manager.index -}}
{{- $agent := .Values.agent -}}
{{- $lb := .Values.gateway.lb -}}
{{- $index := .Values.manager.index -}}
{{- $corrector := .Values.manager.index.corrector -}}
{{- $creator := .Values.manager.index.creator -}}
{{- $saver := .Values.manager.index.saver -}}
{{- if .Values.defaults.networkPolicy.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
Expand Down Expand Up @@ -51,6 +53,18 @@ spec:
podSelector:
matchLabels:
app: {{ $corrector.name }}
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Release.Namespace }}
podSelector:
matchLabels:
app: {{ $creator.name }}
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Release.Namespace }}
podSelector:
matchLabels:
app: {{ $saver.name }}
{{- if .Values.defaults.networkPolicy.custom.ingress }}
{{- toYaml .Values.defaults.networkPolicy.custom.ingress | nindent 4 }}
{{- end }}
Expand Down
20 changes: 17 additions & 3 deletions charts/vald/templates/discoverer/networkpolicy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
#

{{- $discoverer := .Values.discoverer -}}
{{- $lb := .Values.gateway.lb -}}
{{- $index := .Values.manager.index -}}
{{- $corrector := .Values.manager.index.corrector -}}
{{- $lb := .Values.gateway.lb -}}
{{- $index := .Values.manager.index -}}
{{- $corrector := .Values.manager.index.corrector -}}
{{- $creator := .Values.manager.index.creator -}}
{{- $saver := .Values.manager.index.saver -}}
{{- if .Values.defaults.networkPolicy.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
Expand Down Expand Up @@ -53,6 +55,18 @@ spec:
podSelector:
matchLabels:
app: {{ $corrector.name }}
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Release.Namespace }}
podSelector:
matchLabels:
app: {{ $creator.name }}
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Release.Namespace }}
podSelector:
matchLabels:
app: {{ $saver.name }}
{{- if .Values.defaults.networkPolicy.custom.ingress }}
{{- toYaml .Values.defaults.networkPolicy.custom.ingress | nindent 4 }}
{{- end }}
Expand Down
4 changes: 2 additions & 2 deletions charts/vald/templates/index/job/correction/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,12 @@ data:
agent_namespace: {{ $index.indexer.agent_namespace | quote }}
node_name: {{ $index.indexer.node_name | quote }}
stream_list_concurrency: {{ $corrector.stream_list_concurrency }}
bbolt_async_write_concurrency: {{ $corrector.bbolt_async_write_concurrency }}
kvs_async_write_concurrency: {{ $corrector.kvs_async_write_concurrency }}
index_replica: {{ $gateway.gateway_config.index_replica }}
discoverer:
duration: {{ $corrector.discoverer.duration }}
client:
{{- $discovererClient := $index.indexer.discoverer.client }}
{{- $discovererClient := $index.corrector.discoverer.client }}
{{- $discovererServerPort := $discoverer.server_config.servers.grpc.port }}
{{- $defaultDiscovererHost := printf "%s.%s.svc.cluster.local" $discoverer.name .Release.Namespace }}
{{- $defaultDiscovererPort := default .Values.defaults.server_config.servers.grpc.port $discovererServerPort }}
Expand Down
17 changes: 15 additions & 2 deletions charts/vald/templates/index/job/correction/cronjob.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#

{{- $corrector := .Values.manager.index.corrector -}}
{{- if $corrector.enabled }}
apiVersion: batch/v1
Expand All @@ -28,7 +27,7 @@ metadata:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.Version }}
spec:
schedule: {{ $corrector.schedule }}
schedule: {{ $corrector.schedule | quote }}
concurrencyPolicy: Forbid
suspend: {{ $corrector.suspend }}
startingDeadlineSeconds: {{ $corrector.startingDeadlineSeconds }}
Expand All @@ -39,7 +38,21 @@ spec:
metadata:
labels:
app: {{ $corrector.name }}
annotations:
{{- $pprof := default .Values.defaults.server_config.metrics.pprof $corrector.server_config.metrics.pprof -}}
{{- if $pprof.enabled }}
pyroscope.io/scrape: "true"
pyroscope.io/application-name: {{ $corrector.name }}
pyroscope.io/profile-cpu-enabled: "true"
pyroscope.io/profile-mem-enabled: "true"
pyroscope.io/port: {{ $pprof.port | quote }}
{{- end }}
spec:
{{- if $corrector.initContainers }}
initContainers:
{{- $initContainers := dict "initContainers" $corrector.initContainers "Values" .Values "namespace" .Release.Namespace -}}
{{- include "vald.initContainers" $initContainers | trim | nindent 12 }}
{{- end }}
containers:
- name: {{ $corrector.name }}
image: "{{ $corrector.image.repository }}:{{ default .Values.defaults.image.tag $corrector.image.tag }}"
Expand Down
60 changes: 60 additions & 0 deletions charts/vald/templates/index/job/correction/networkpolicy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#
# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

{{- $agent := .Values.agent -}}
{{- $discoverer := .Values.discoverer -}}
{{- $corrector := .Values.manager.index.corrector -}}
{{- if and .Values.defaults.networkPolicy.enabled $corrector.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ $corrector.name }}-allow
spec:
podSelector:
matchLabels:
app: {{ $corrector.name }}
policyTypes:
- Ingress
- Egress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
{{- if .Values.defaults.networkPolicy.custom.ingress }}
{{- toYaml .Values.defaults.networkPolicy.custom.ingress | nindent 4 }}
{{- end }}
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Release.Namespace }}
podSelector:
matchLabels:
app: {{ $agent.name }}
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Release.Namespace }}
podSelector:
matchLabels:
app: {{ $discoverer.name }}
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
{{- if .Values.defaults.networkPolicy.custom.egress }}
{{- toYaml .Values.defaults.networkPolicy.custom.egress | nindent 4 }}
{{- end }}
{{- end }}
70 changes: 70 additions & 0 deletions charts/vald/templates/index/job/creation/configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#
# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- $creator := .Values.manager.index.creator -}}
{{- $gateway := .Values.gateway.lb -}}
{{- $index := .Values.manager.index -}}
{{- $agent := .Values.agent -}}
{{- $discoverer := .Values.discoverer -}}
{{- if $creator.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $creator.name }}-config
labels:
app.kubernetes.io/name: {{ include "vald.name" . }}
helm.sh/chart: {{ include "vald.chart" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.Version }}
app.kubernetes.io/component: {{ $creator.name }}
data:
config.yaml: |
---
version: {{ $creator.version }}
time_zone: {{ default .Values.defaults.time_zone $creator.time_zone }}
logging:
{{- $logging := dict "Values" $creator.logging "default" .Values.defaults.logging }}
{{- include "vald.logging" $logging | nindent 6 }}
server_config:
{{- $servers := dict "Values" $creator.server_config "default" .Values.defaults.server_config }}
{{- include "vald.servers" $servers | nindent 6 }}
observability:
{{- $observability := dict "Values" $creator.observability "default" .Values.defaults.observability }}
{{- include "vald.observability" $observability | nindent 6 }}
creator:
agent_port: {{ default .Values.defaults.server_config.servers.grpc.port $agent.server_config.servers.grpc.port }}
agent_name: {{ $agent.name | quote }}
agent_dns: {{ $agent.name }}.{{ .Release.Namespace }}.svc.cluster.local
agent_namespace: {{ $creator.agent_namespace | quote }}
node_name: {{ $creator.node_name | quote }}
concurrency: {{ $creator.concurrency }}
target_addrs: {{ $creator.target_addrs }}
discoverer:
duration: {{ $creator.discoverer.duration }}
client:
{{- $discovererClient := $creator.discoverer.client }}
{{- $discovererServerPort := $discoverer.server_config.servers.grpc.port }}
{{- $defaultDiscovererHost := printf "%s.%s.svc.cluster.local" $discoverer.name .Release.Namespace }}
{{- $defaultDiscovererPort := default .Values.defaults.server_config.servers.grpc.port $discovererServerPort }}
{{- $defaultDiscovererAddr := (list (printf "%s:%d" $defaultDiscovererHost (int64 $defaultDiscovererPort))) }}
{{- $discovererAddrs := dict "Values" $discovererClient.addrs "default" $defaultDiscovererAddr }}
{{- include "vald.grpc.client.addrs" $discovererAddrs | nindent 10 }}
{{- $discovererGRPCclient := dict "Values" $discovererClient "default" .Values.defaults.grpc.client }}
{{- include "vald.grpc.client" $discovererGRPCclient | nindent 10 }}
agent_client_options:
{{- include "vald.grpc.client.addrs" (dict "Values" $creator.discoverer.agent_client_options.addrs) | nindent 10 }}
{{- include "vald.grpc.client" (dict "Values" $creator.discoverer.agent_client_options "default" .Values.defaults.grpc.client) | nindent 10 }}
{{- end }}
78 changes: 78 additions & 0 deletions charts/vald/templates/index/job/creation/cronjob.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#
# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{{- $creator := .Values.manager.index.creator -}}
{{- if $creator.enabled }}
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ $creator.name }}
labels:
app: {{ $creator.name }}
app.kubernetes.io/name: {{ include "vald.name" . }}
helm.sh/chart: {{ include "vald.chart" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/version: {{ .Chart.Version }}
spec:
schedule: {{ $creator.schedule | quote }}
concurrencyPolicy: Forbid
suspend: {{ $creator.suspend }}
startingDeadlineSeconds: {{ $creator.startingDeadlineSeconds }}
jobTemplate:
spec:
ttlSecondsAfterFinished: {{ $creator.ttlSecondsAfterFinished }}
template:
metadata:
labels:
app: {{ $creator.name }}
app.kubernetes.io/name: {{ include "vald.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: {{ $creator.name }}
annotations:
{{- $pprof := default .Values.defaults.server_config.metrics.pprof $creator.server_config.metrics.pprof -}}
{{- if $pprof.enabled }}
pyroscope.io/scrape: "true"
pyroscope.io/application-name: {{ $creator.name }}
pyroscope.io/profile-cpu-enabled: "true"
pyroscope.io/profile-mem-enabled: "true"
pyroscope.io/port: {{ $pprof.port | quote }}
{{- end }}
spec:
{{- if $creator.initContainers }}
initContainers:
{{- $initContainers := dict "initContainers" $creator.initContainers "Values" .Values "namespace" .Release.Namespace -}}
{{- include "vald.initContainers" $initContainers | trim | nindent 12 }}
{{- end }}
containers:
- name: {{ $creator.name }}
image: "{{ $creator.image.repository }}:{{ default .Values.defaults.image.tag $creator.image.tag }}"
imagePullPolicy: {{ $creator.image.pullPolicy }}
volumeMounts:
- name: {{ $creator.name }}-config
mountPath: /etc/server/
{{- $servers := dict "Values" $creator.server_config "default" .Values.defaults.server_config -}}
{{- include "vald.containerPorts" $servers | trim | nindent 14 }}
{{- if $creator.env }}
env:
{{- toYaml $creator.env | nindent 16 }}
{{- end }}
restartPolicy: OnFailure
volumes:
- name: {{ $creator.name }}-config
configMap:
defaultMode: 420
name: {{ $creator.name }}-config
{{- end }}
Loading

0 comments on commit 1625346

Please sign in to comment.