Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cronjob for readreplica rotator #2242

Merged
merged 47 commits into from
Dec 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
3767c8b
Implement sample read replica rotation logic
ykadowak Nov 10, 2023
1e4a070
Add external-snapshotter in go.mod
ykadowak Nov 16, 2023
82c4d11
Add readreplica initial implementation
ykadowak Nov 16, 2023
875ce3b
Add service implementation of readreplica rotator
ykadowak Nov 20, 2023
9336760
Add readreplica rotate config
ykadowak Nov 20, 2023
747cf92
Refactor to read labels
ykadowak Nov 21, 2023
57812e2
replace id to _MY_TARGET_REPLICA_ID_
ykadowak Nov 22, 2023
4a48422
Add snapshot k8s client
ykadowak Nov 22, 2023
45cc8e4
Format
ykadowak Nov 22, 2023
bb14dab
Fix snapshot client initialization
ykadowak Nov 22, 2023
162a786
Add Apache License to main.go
ykadowak Nov 22, 2023
9a93754
Use GetConfigOrDie
ykadowak Nov 22, 2023
eed7522
Use internal/k8s client
ykadowak Nov 22, 2023
1fdd3d2
Refactor
ykadowak Nov 22, 2023
b6f93f2
Use controller-runtime for CRUD in readreplica
ykadowak Nov 24, 2023
b6d4580
Remove snapshot client and use controller-runtime
ykadowak Nov 24, 2023
945fe76
Move LabelSelector into internal/k8s
ykadowak Nov 24, 2023
79f7de5
Report error to span
ykadowak Nov 24, 2023
1c982ca
Merge branch 'main' into feature/example/readreplica
ykadowak Nov 24, 2023
167e704
Update go modules
ykadowak Nov 24, 2023
b4051ae
Remove old example
ykadowak Nov 24, 2023
2a5d358
Refactor
ykadowak Nov 24, 2023
7b3f428
Disable exhaustruct for now
ykadowak Nov 24, 2023
55e9b6a
Fix predeclared
ykadowak Nov 24, 2023
bebb04c
Fix stylecheck
ykadowak Nov 24, 2023
3e6f179
nolint:gomnd
ykadowak Nov 24, 2023
f212824
Add test template
ykadowak Nov 24, 2023
b5897c2
Add test for getNewBaseName
ykadowak Nov 24, 2023
cebb8c6
Add readreplica-rotate build scripts
ykadowak Nov 27, 2023
228a4a5
Add service account and clusterroles for read replica rotator
ykadowak Nov 28, 2023
484bb39
Merge branch 'main' into feature/readreplica/job
ykadowak Nov 28, 2023
21e93a2
Update charts
ykadowak Nov 28, 2023
d967ef6
Update comment
ykadowak Nov 28, 2023
c8b9760
Merge branch 'main' into feature/readreplica/job
ykadowak Nov 29, 2023
a0575ba
update charts
ykadowak Nov 29, 2023
4256774
Update docker image name in Makefile.d/docker.mk
ykadowak Nov 29, 2023
5f6af0c
Fix scheme
ykadowak Nov 29, 2023
ee77443
Update docs
ykadowak Nov 29, 2023
6dba952
Fix comment
ykadowak Nov 29, 2023
f666aed
style: format code with Gofumpt and Prettier
deepsource-autofix[bot] Nov 29, 2023
372f172
Merge branch 'main' into feature/readreplica/job
ykadowak Dec 4, 2023
cb12bcc
Merge branch 'main' into feature/readreplica/job
ykadowak Dec 5, 2023
3ec66bf
Fix indent
ykadowak Dec 5, 2023
6544457
Add network policy for readreplica rotator
ykadowak Dec 6, 2023
fb72e4b
Merge branch 'main' into feature/readreplica/job
ykadowak Dec 6, 2023
23ed2ed
Merge remote-tracking branch 'origin/main' into feature/readreplica/job
ykadowak Dec 6, 2023
5c480d7
Update README.md
ykadowak Dec 6, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions .github/workflows/dockers-readreplica-rotate.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#
# Copyright (C) 2019-2023 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Build docker image: readreplica-rotate"
on:
push:
branches:
- main
tags:
- "*.*.*"
- "v*.*.*"
- "*.*.*-*"
- "v*.*.*-*"
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/dockers-readreplica-rotate.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/index/job/readreplica/rotate/**"
- "cmd/index/job/readreplica/rotate/**"
- "dockers/index/job/readreplica/rotate/Dockerfile"
- "versions/GO_VERSION"
pull_request:
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-readreplica-rotate.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/index/job/readreplica/rotate/**"
- "cmd/index/job/readreplica/rotate/**"
- "dockers/index/job/readreplica/rotate/Dockerfile"
- "versions/GO_VERSION"
pull_request_target:
paths:
- ".github/actions/docker-build/actions.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-readreplica-rotate.yml"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/index/job/readreplica/rotate/**"
- "cmd/index/job/readreplica/rotate/**"
- "dockers/index/job/readreplica/rotate/Dockerfile"
- "versions/GO_VERSION"

jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
with:
target: readreplica-rotate
secrets: inherit
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ LOADTEST_IMAGE = $(NAME)-loadtest
INDEX_CORRECTION_IMAGE = $(NAME)-index-correction
INDEX_CREATION_IMAGE = $(NAME)-index-creation
INDEX_SAVE_IMAGE = $(NAME)-index-save
READREPLICA_ROTATE_IMAGE = $(NAME)-readreplica-rotate
MANAGER_INDEX_IMAGE = $(NAME)-manager-index
MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"

Expand Down
29 changes: 29 additions & 0 deletions Makefile.d/build.mk
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,35 @@ cmd/index/job/save/index-save: \
$(dir $@)main.go
$@ -version

cmd/index/job/readreplica/rotate/readreplica-rotate: \
$(GO_SOURCES_INTERNAL) \
$(PBGOS) \
$(shell find $(ROOTDIR)/cmd/index/job/readreplica/rotate -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
$(shell find $(ROOTDIR)/pkg/index/job/readreplica/rotate -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
$(eval CGO_ENABLED = 0)
CGO_ENABLED=$(CGO_ENABLED) \
GO111MODULE=on \
GOPRIVATE=$(GOPRIVATE) \
go build \
--ldflags "-w -extldflags=-static \
-X '$(GOPKG)/internal/info.Version=$(VERSION)' \
-X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \
-X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \
-X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \
-X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \
-X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \
-X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \
-X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \
-buildid=" \
-mod=readonly \
-modcacherw \
-a \
-tags "osusergo netgo static_build" \
-trimpath \
-o $@ \
$(dir $@)main.go
$@ -version

.PHONY: binary/build/zip
## build all binaries and zip them
binary/build/zip: \
Expand Down
14 changes: 14 additions & 0 deletions Makefile.d/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -230,3 +230,17 @@ docker/build/index-save:
-t $(ORG)/$(INDEX_SAVE_IMAGE):$(TAG) . \
--build-arg MAINTAINER=$(MAINTAINER) \
--build-arg GO_VERSION=$(GO_VERSION)

.PHONY: docker/name/readreplica-rotate
docker/name/readreplica-rotate:
@echo "$(ORG)/$(READREPLICA_ROTATE_IMAGE)"

.PHONY: docker/build/readreplica-rotate
## build readreplica-rotate image
docker/build/readreplica-rotate:
$(DOCKER) build \
$(DOCKER_OPTS) \
-f dockers/index/job/readreplica/rotate/Dockerfile \
-t $(ORG)/$(READREPLICA_ROTATE_IMAGE):$(TAG) . \
--build-arg MAINTAINER=$(MAINTAINER) \
--build-arg GO_VERSION=$(GO_VERSION)
8 changes: 8 additions & 0 deletions Makefile.d/k8s.mk
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,15 @@ k8s/manifest/update: \
mkdir -p k8s/gateway
mkdir -p k8s/manager
mkdir -p k8s/index/job
mkdir -p k8s/index/job/readreplica
mv $(TEMP_DIR)/vald/templates/agent k8s/agent
mv $(TEMP_DIR)/vald/templates/discoverer k8s/discoverer
mv $(TEMP_DIR)/vald/templates/gateway/lb k8s/gateway/lb
mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index
mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction
mv $(TEMP_DIR)/vald/templates/index/job/creation k8s/index/job/creation
mv $(TEMP_DIR)/vald/templates/index/job/save k8s/index/job/save
mv $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate k8s/index/job/readreplica/rotate
rm -rf $(TEMP_DIR)

.PHONY: k8s/manifest/helm-operator/clean
Expand Down Expand Up @@ -87,6 +89,9 @@ k8s/vald/deploy:
kubectl apply -f $(TEMP_DIR)/vald/templates/discoverer || true
kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/lb || true
kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/correction || true
kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/creation || true
kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/save || true
kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate || true
rm -rf $(TEMP_DIR)
kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n"

Expand All @@ -104,6 +109,9 @@ k8s/vald/delete:
--set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \
--output-dir $(TEMP_DIR) \
charts/vald
kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate
kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/save
kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/creation
kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction
kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb
kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index
Expand Down
Loading
Loading