diff --git a/.github/workflows/dockers-readreplica-rotate.yml b/.github/workflows/dockers-readreplica-rotate.yml new file mode 100644 index 00000000000..74eabf441d2 --- /dev/null +++ b/.github/workflows/dockers-readreplica-rotate.yml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: readreplica-rotate" +on: + push: + branches: + - main + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-readreplica-rotate.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/readreplica/rotate/**" + - "cmd/index/job/readreplica/rotate/**" + - "dockers/index/job/readreplica/rotate/Dockerfile" + - "versions/GO_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-readreplica-rotate.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/readreplica/rotate/**" + - "cmd/index/job/readreplica/rotate/**" + - "dockers/index/job/readreplica/rotate/Dockerfile" + - "versions/GO_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-readreplica-rotate.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/readreplica/rotate/**" + - "cmd/index/job/readreplica/rotate/**" + - "dockers/index/job/readreplica/rotate/Dockerfile" + - "versions/GO_VERSION" + +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: readreplica-rotate + secrets: inherit diff --git a/Makefile b/Makefile index 4a82f747b6d..de6f1b3cce5 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ LOADTEST_IMAGE = $(NAME)-loadtest INDEX_CORRECTION_IMAGE = $(NAME)-index-correction INDEX_CREATION_IMAGE = $(NAME)-index-creation INDEX_SAVE_IMAGE = $(NAME)-index-save +READREPLICA_ROTATE_IMAGE = $(NAME)-readreplica-rotate MANAGER_INDEX_IMAGE = $(NAME)-manager-index MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>" diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index 8bc4fbb4449..0911abaf4fb 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -293,6 +293,35 @@ cmd/index/job/save/index-save: \ $(dir $@)main.go $@ -version +cmd/index/job/readreplica/rotate/readreplica-rotate: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find $(ROOTDIR)/cmd/index/job/readreplica/rotate -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find $(ROOTDIR)/pkg/index/job/readreplica/rotate -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + $(eval CGO_ENABLED = 0) + CGO_ENABLED=$(CGO_ENABLED) \ + GO111MODULE=on \ + GOPRIVATE=$(GOPRIVATE) \ + go build \ + --ldflags "-w -extldflags=-static \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ + -buildid=" \ + -mod=readonly \ + -modcacherw \ + -a \ + -tags "osusergo netgo static_build" \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + $@ -version + .PHONY: binary/build/zip ## build all binaries and zip them binary/build/zip: \ diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 7c1bb74e7c7..9bf94caf52f 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -230,3 +230,17 @@ docker/build/index-save: -t $(ORG)/$(INDEX_SAVE_IMAGE):$(TAG) . \ --build-arg MAINTAINER=$(MAINTAINER) \ --build-arg GO_VERSION=$(GO_VERSION) + +.PHONY: docker/name/readreplica-rotate +docker/name/readreplica-rotate: + @echo "$(ORG)/$(INDEX_SAVE_IMAGE)" + +.PHONY: docker/build/readreplica-rotate +## build readreplica-rotate image +docker/build/readreplica-rotate: + $(DOCKER) build \ + $(DOCKER_OPTS) \ + -f dockers/index/job/readreplica/rotate/Dockerfile \ + -t $(ORG)/$(READREPLICA_ROTATE_IMAGE):$(TAG) . \ + --build-arg MAINTAINER=$(MAINTAINER) \ + --build-arg GO_VERSION=$(GO_VERSION) diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index 6ee28a3bae4..f8b93a35b73 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -45,6 +45,7 @@ k8s/manifest/update: \ mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction mv $(TEMP_DIR)/vald/templates/index/job/creation k8s/index/job/creation mv $(TEMP_DIR)/vald/templates/index/job/save k8s/index/job/save + mv $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate k8s/index/job/readreplica/rotate rm -rf $(TEMP_DIR) .PHONY: k8s/manifest/helm-operator/clean @@ -87,6 +88,9 @@ k8s/vald/deploy: kubectl apply -f $(TEMP_DIR)/vald/templates/discoverer || true kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/lb || true kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/correction || true + kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/creation || true + kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/save || true + kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate || true rm -rf $(TEMP_DIR) kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n" @@ -104,6 +108,9 @@ k8s/vald/delete: --set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \ --output-dir $(TEMP_DIR) \ charts/vald + kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate + kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save + kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index diff --git a/charts/vald/templates/index/job/readreplica/rotate/configmap.yaml b/charts/vald/templates/index/job/readreplica/rotate/configmap.yaml new file mode 100644 index 00000000000..6402c217b06 --- /dev/null +++ b/charts/vald/templates/index/job/readreplica/rotate/configmap.yaml @@ -0,0 +1,52 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $rotator := .Values.manager.index.readreplica.rotator -}} +{{- $gateway := .Values.gateway.lb -}} +{{- $index := .Values.manager.index -}} +{{- $agent := .Values.agent -}} +{{- $discoverer := .Values.discoverer -}} +{{- if $rotator.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $rotator.name }}-config + labels: + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} + app.kubernetes.io/component: {{ $rotator.name }} +data: + config.yaml: | + --- + version: {{ $rotator.version }} + time_zone: {{ default .Values.defaults.time_zone $rotator.time_zone }} + logging: + {{- $logging := dict "Values" $rotator.logging "default" .Values.defaults.logging }} + {{- include "vald.logging" $logging | nindent 6 }} + server_config: + {{- $servers := dict "Values" $rotator.server_config "default" .Values.defaults.server_config }} + {{- include "vald.servers" $servers | nindent 6 }} + observability: + {{- $observability := dict "Values" $rotator.observability "default" .Values.defaults.observability }} + {{- include "vald.observability" $observability | nindent 6 }} + rotator: + agent_namespace: {{ $rotator.agent_namespace | quote }} + read_replica_label_key: "vald-readreplica-id" + read_replica_id: "_MY_TARGET_REPLICA_ID_" + volume_name: "vald-agent-ngt-readreplica-pvc" +{{- end }} diff --git a/charts/vald/templates/index/job/readreplica/rotate/cronjob.yaml b/charts/vald/templates/index/job/readreplica/rotate/cronjob.yaml new file mode 100644 index 00000000000..21bc4414755 --- /dev/null +++ b/charts/vald/templates/index/job/readreplica/rotate/cronjob.yaml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $rotator := .Values.manager.index.readreplica.rotator -}} +{{- if $rotator.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ $rotator.name }} + labels: + app: {{ $rotator.name }} + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} +spec: + schedule: "0 0 31 2 *" # This never happens as February never has 31 days so we can prevent this cronjob from running automatically + concurrencyPolicy: Forbid + suspend: true # This cronjob should only be run manually by index manager + startingDeadlineSeconds: 0 # To prevent being blocked in any case + jobTemplate: + spec: + ttlSecondsAfterFinished: {{ $rotator.ttlSecondsAfterFinished }} + template: + metadata: + labels: + app: {{ $rotator.name }} + app.kubernetes.io/name: {{ include "vald.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: {{ $rotator.name }} + annotations: + {{- $pprof := default .Values.defaults.server_config.metrics.pprof $rotator.server_config.metrics.pprof -}} + {{- if $pprof.enabled }} + pyroscope.io/scrape: "true" + pyroscope.io/application-name: {{ $rotator.name }} + pyroscope.io/profile-cpu-enabled: "true" + pyroscope.io/profile-mem-enabled: "true" + pyroscope.io/port: {{ $pprof.port | quote }} + {{- end }} + spec: + {{- if $rotator.initContainers }} + initContainers: + {{- $initContainers := dict "initContainers" $rotator.initContainers "Values" .Values "namespace" .Release.Namespace -}} + {{- include "vald.initContainers" $initContainers | trim | nindent 12 }} + {{- end }} + containers: + - name: {{ $rotator.name }} + image: "{{ $rotator.image.repository }}:{{ default .Values.defaults.image.tag $rotator.image.tag }}" + imagePullPolicy: {{ $rotator.image.pullPolicy }} + volumeMounts: + - name: {{ $rotator.name }}-config + mountPath: /etc/server/ + {{- $servers := dict "Values" $rotator.server_config "default" .Values.defaults.server_config -}} + {{- include "vald.containerPorts" $servers | trim | nindent 14 }} + {{- if $rotator.env }} + env: + {{- toYaml $rotator.env | nindent 16 }} + {{- end }} + restartPolicy: OnFailure + volumes: + - name: {{ $rotator.name }}-config + configMap: + defaultMode: 420 + name: {{ $rotator.name }}-config +{{- end }} diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index b843b406731..4e37e2e9ba1 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -2946,3 +2946,67 @@ manager: net: dialer: keepalive: 15m #indexer fetches uncommitted index length, which includes huge payload so we need to set keepalive longer than usual + readreplica: + # @schema {"name": "manager.index.readreplica.rotator", "type": "object"} + rotator: + # @schema {"name": "manager.index.readreplica.rotator.name", "type": "string"} + # manager.index.readreplica.rotator.name -- name of index correction job + name: vald-readreplica-rotate + # @schema {"name": "manager.index.readreplica.rotator.image", "alias": "image"} + image: + # manager.index.readreplica.rotator.image.repository -- image repository + repository: vdaas/vald-readreplica-rotate + # manager.index.readreplica.rotator.image.tag -- image tag (overrides defaults.image.tag) + tag: "" + # manager.index.image.pullPolicy -- image pull policy + pullPolicy: Always + # @schema {"name": "manager.index.readreplica.rotator.server_config", "alias": "server_config"} + # manager.index.readreplica.rotator.server_config -- server config (overrides defaults.server_config) + server_config: + servers: + rest: {} + grpc: {} + healths: + liveness: {} + readiness: {} + startup: {} + metrics: + pprof: {} + # @schema {"name": "manager.index.readreplica.rotator.initContainers", "alias": "initContainers"} + # manager.index.readreplica.rotator.initContainers -- init containers + initContainers: [] + # @schema {"name": "manager.index.readreplica.rotator.env", "alias": "env"} + # manager.index.readreplica.rotator.env -- environment variables + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_READREPLICA_ID + value: 0 + # @schema {"name": "manager.index.readreplica.rotator.observability", "alias": "observability"} + # manager.index.readreplica.rotator.observability -- observability config (overrides defaults.observability) + observability: + otlp: + attribute: + service_name: vald-readreplica-rotation + # @schema {"name": "manager.index.readreplica.rotator.ttlSecondsAfterFinished", "type": "integer"} + # manager.index.readreplica.rotator.ttlSecondsAfterFinished -- ttl setting for K8s completed jobs + ttlSecondsAfterFinished: 86400 + # @schema {"name": "manager.index.readreplica.rotator.version", "alias": "version"} + # manager.index.readreplica.rotator.version -- version of readreplica rotator config + version: v0.0.0 + # @schema {"name": "manager.index.readreplica.rotator.agent_namespace", "type": "string"} + # manager.index.readreplica.rotator.agent_namespace -- namespace of agent pods to manage + agent_namespace: _MY_POD_NAMESPACE_ + # @schema {"name": "manager.index.readreplica.rotator.read_replica_id", "type": "string"} + # manager.index.readreplica.rotator.read_replica_id -- read replica id to perform rotation + read_replica_id: _MY_TARGET_REPLICA_ID_ diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile new file mode 100644 index 00000000000..20531bf9742 --- /dev/null +++ b/dockers/index/job/readreplica/rotate/Dockerfile @@ -0,0 +1,93 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG GO_VERSION=latest +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG MAINTAINER="vdaas.org vald team " + +FROM golang:${GO_VERSION} AS golang + +FROM ubuntu:devel AS builder + +ENV GO111MODULE on +ENV DEBIAN_FRONTEND noninteractive +ENV INITRD No +ENV LANG en_US.UTF-8 +ENV GOROOT /opt/go +ENV GOPATH /go +ENV PATH ${PATH}:${GOROOT}/bin:${GOPATH}/bin +ENV ORG vdaas +ENV REPO vald +ENV PKG index/job/readreplica/rotate +ENV APP_NAME readreplica-rotate + +# skipcq: DOK-DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + curl \ + upx \ + git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=golang /usr/local/go $GOROOT +RUN mkdir -p "$GOPATH/src" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY Makefile . +COPY .git . +COPY go.mod . +COPY go.sum . + +RUN make go/download + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal +COPY internal . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc +COPY apis/grpc . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} +COPY pkg/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +COPY cmd/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +RUN cp sample.yaml /tmp/config.yaml + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer="${MAINTAINER}" + +ENV APP_NAME readreplica-rotate + +COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +COPY --from=builder /tmp/config.yaml /etc/server/config.yaml + +USER nonroot:nonroot + +ENTRYPOINT ["/go/bin/readreplica-rotate"]