From f68adfa02ebcf5c7ccd9f8f102fe759d86ffed4e Mon Sep 17 00:00:00 2001 From: Kiichiro YUKAWA Date: Tue, 23 Jan 2024 11:30:13 +0900 Subject: [PATCH 01/10] Add go cache for improvement docker build performance (#2297) * :whale: Add go cache for improvement docker build performance Signed-off-by: vankichi * :whale: Fix Signed-off-by: vankichi * :whale: Fix Signed-off-by: vankichi * :whale: Add skip clean deps flag Signed-off-by: vankichi * :whale: Fix Signed-off-by: vankichi * :whale: Fix Signed-off-by: vankichi * :whale: remove Signed-off-by: vankichi * :whale: convert /tmp to /home/vankichi Signed-off-by: vankichi --------- Signed-off-by: vankichi --- Makefile | 1 + Makefile.d/dependencies.mk | 29 ++++++++++--------- dockers/agent/core/ngt/Dockerfile | 11 +++++-- dockers/agent/sidecar/Dockerfile | 11 +++++-- dockers/ci/base/Dockerfile | 11 +++++-- dockers/dev/Dockerfile | 12 ++++++-- dockers/discoverer/k8s/Dockerfile | 11 +++++-- dockers/gateway/filter/Dockerfile | 11 +++++-- dockers/gateway/lb/Dockerfile | 11 +++++-- dockers/gateway/mirror/Dockerfile | 11 +++++-- dockers/index/job/correction/Dockerfile | 11 +++++-- dockers/index/job/creation/Dockerfile | 11 +++++-- .../index/job/readreplica/rotate/Dockerfile | 11 +++++-- dockers/index/job/save/Dockerfile | 11 +++++-- dockers/manager/index/Dockerfile | 11 +++++-- dockers/tools/benchmark/job/Dockerfile | 10 +++++-- dockers/tools/benchmark/operator/Dockerfile | 11 +++++-- dockers/tools/cli/loadtest/Dockerfile | 7 ++++- 18 files changed, 158 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 7d6227297c..103b2b8508 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,7 @@ GOARCH := $(eval GOARCH := $(shell go env GOARCH))$(GOARCH) GOBIN := $(eval GOBIN := $(or $(shell go env GOBIN),$(GOPATH)/bin))$(GOBIN) GOCACHE := $(eval GOCACHE := $(shell go env GOCACHE))$(GOCACHE) GOOS := $(eval GOOS := $(shell go env GOOS))$(GOOS) +GO_CLEAN_DEPS := true GOTEST_TIMEOUT = 30m TEST_NOT_IMPL_PLACEHOLDER = NOT IMPLEMENTED BELOW diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index f423031547..1969ff2756 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -47,19 +47,22 @@ go/download: .PHONY: go/deps ## install Go package dependencies go/deps: - rm -rf $(ROOTDIR)/vendor \ - /go/pkg \ - $(GOCACHE) \ - $(ROOTDIR)/go.sum \ - $(ROOTDIR)/go.mod - cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod - GOPRIVATE=$(GOPRIVATE) go mod tidy - go clean -cache -modcache -testcache -i -r - rm -rf $(ROOTDIR)/vendor \ - /go/pkg \ - $(GOCACHE) \ - $(ROOTDIR)/go.sum \ - $(ROOTDIR)/go.mod + if $(GO_CLEAN_DEPS); then \ + rm -rf $(ROOTDIR)/vendor \ + /go/pkg \ + $(GOCACHE) \ + $(ROOTDIR)/go.sum \ + $(ROOTDIR)/go.mod ; \ + cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod ; \ + GOPRIVATE=$(GOPRIVATE) go mod tidy ; \ + go clean -cache -modcache -testcache -i -r ; \ + rm -rf $(ROOTDIR)/vendor \ + /go/pkg \ + $(GOCACHE) \ + $(ROOTDIR)/go.sum \ + $(ROOTDIR)/go.mod ; \ + cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod ; \ + fi cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod GOPRIVATE=$(GOPRIVATE) go mod tidy go get -u all 2>/dev/null || true diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index 5670b1baf2..a13e92d0bd 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -65,7 +68,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -87,7 +92,9 @@ COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} RUN make ngt/install -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index 46fe904f2f..c602fbd43b 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV GOPATH /go ENV GOROOT /opt/go @@ -58,7 +61,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -79,7 +84,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" FROM --platform=${BUILDPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index d60dd98cf7..edf2db83d7 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -23,6 +23,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder LABEL maintainer="${MAINTAINER}" +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -71,9 +74,13 @@ COPY apis/proto apis/proto COPY versions versions COPY hack/go.mod.default hack/go.mod.default -RUN make deps ROOTDIR=$ROOTDIR +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make deps ROOTDIR=$ROOTDIR GO_CLEAN_DEPS=false -RUN make ngt/install \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make ngt/install \ && make helm/install \ && make helm-docs/install \ && make kind/install \ diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index 166285a93d..00ae3fa9da 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -21,6 +21,10 @@ ARG MAINTAINER="vdaas.org vald team " FROM --platform=${BUILDPLATFORM} mcr.microsoft.com/vscode/devcontainers/go:1 AS base LABEL maintainer="${MAINTAINER}" +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} + # skipcq: DOK-DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -51,7 +55,9 @@ COPY versions versions COPY hack/go.mod.default hack/go.mod.default # basic deps -RUN make deps \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make deps GO_CLEAN_DEPS=false \ && make ngt/install \ && make helm/install \ && make helm-docs/install \ @@ -62,7 +68,9 @@ RUN make deps \ && make kubectl/install # additional deps -RUN make k3d/install \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make k3d/install \ && make buf/install \ && make k9s/install \ && make minikube/install diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile index 788c40ffe5..d6819fe551 100644 --- a/dockers/discoverer/k8s/Dockerfile +++ b/dockers/discoverer/k8s/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -58,7 +61,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -76,7 +81,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile index 2f6717ad3e..23dcafa1dd 100644 --- a/dockers/gateway/filter/Dockerfile +++ b/dockers/gateway/filter/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile index f9d872d8b8..d54cd7b49d 100644 --- a/dockers/gateway/lb/Dockerfile +++ b/dockers/gateway/lb/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile index 438790483a..d146917731 100644 --- a/dockers/gateway/mirror/Dockerfile +++ b/dockers/gateway/mirror/Dockerfile @@ -23,6 +23,9 @@ FROM golang:${GO_VERSION} AS golang FROM ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -53,7 +56,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY go.mod . COPY go.sum . -RUN go mod download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + go mod download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -77,7 +82,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . COPY .git . -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile index 5625c1193e..bb7c3e3d3c 100644 --- a/dockers/index/job/correction/Dockerfile +++ b/dockers/index/job/correction/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile index daa8bb6c79..a0bd366138 100644 --- a/dockers/index/job/creation/Dockerfile +++ b/dockers/index/job/creation/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile index 787405e48e..36e19db46f 100644 --- a/dockers/index/job/readreplica/rotate/Dockerfile +++ b/dockers/index/job/readreplica/rotate/Dockerfile @@ -24,6 +24,9 @@ FROM golang:${GO_VERSION} AS golang FROM ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile index 7c543d8744..6b5a6beb39 100644 --- a/dockers/index/job/save/Dockerfile +++ b/dockers/index/job/save/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile index 0c5457160d..72a77f0cd7 100644 --- a/dockers/manager/index/Dockerfile +++ b/dockers/manager/index/Dockerfile @@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index 6a3111f26d..3d678ee7ba 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -30,7 +30,9 @@ FROM ubuntu:devel AS builder ARG UPX_OPTIONS ARG ZLIB_VERSION ARG HDF5_VERSION +ARG TARGETARCH +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -88,7 +90,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -106,7 +110,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/tools/benchmark/operator/Dockerfile b/dockers/tools/benchmark/operator/Dockerfile index ba3dc24266..22e298b3f2 100644 --- a/dockers/tools/benchmark/operator/Dockerfile +++ b/dockers/tools/benchmark/operator/Dockerfile @@ -25,6 +25,9 @@ FROM golang:${GO_VERSION} AS golang FROM ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -62,7 +65,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN go mod download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . @@ -80,7 +85,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile index b086da97db..22c181fc9c 100644 --- a/dockers/tools/cli/loadtest/Dockerfile +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -22,6 +22,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder +ARG TARGETARCH + +ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -57,7 +60,9 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN make go/download +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ + make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal COPY internal . From 07570a1ff28780ddf25d7fc7db8cc548b88898f4 Mon Sep 17 00:00:00 2001 From: Yusuke Kadowaki Date: Tue, 23 Jan 2024 14:49:14 +0900 Subject: [PATCH 02/10] Stop using ENV ARCH and add --platform in Dockerfile (#2304) * Stop using ENV ARCH * Update Dockerfile platform specification --- dockers/agent/core/ngt/Dockerfile | 9 ++++----- dockers/agent/sidecar/Dockerfile | 9 ++++----- dockers/ci/base/Dockerfile | 9 ++++----- dockers/dev/Dockerfile | 10 ++++------ dockers/discoverer/k8s/Dockerfile | 9 ++++----- dockers/gateway/filter/Dockerfile | 9 ++++----- dockers/gateway/lb/Dockerfile | 9 ++++----- dockers/gateway/mirror/Dockerfile | 15 +++++++-------- dockers/index/job/correction/Dockerfile | 9 ++++----- dockers/index/job/creation/Dockerfile | 9 ++++----- dockers/index/job/readreplica/rotate/Dockerfile | 15 +++++++-------- dockers/index/job/save/Dockerfile | 9 ++++----- dockers/manager/index/Dockerfile | 9 ++++----- dockers/tools/benchmark/job/Dockerfile | 15 +++++++-------- dockers/tools/benchmark/operator/Dockerfile | 15 +++++++-------- dockers/tools/cli/loadtest/Dockerfile | 5 ++--- 16 files changed, 74 insertions(+), 91 deletions(-) diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index a13e92d0bd..d98150a793 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -68,8 +67,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -92,8 +91,8 @@ COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} RUN make ngt/install -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index c602fbd43b..7784edf21a 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV GOPATH /go ENV GOROOT /opt/go @@ -61,8 +60,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -84,8 +83,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index edf2db83d7..2bb5787508 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -25,7 +25,6 @@ LABEL maintainer="${MAINTAINER}" ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -74,12 +73,12 @@ COPY apis/proto apis/proto COPY versions versions COPY hack/go.mod.default hack/go.mod.default -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make deps ROOTDIR=$ROOTDIR GO_CLEAN_DEPS=false -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make ngt/install \ && make helm/install \ && make helm-docs/install \ diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index 00ae3fa9da..7a33529525 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -23,8 +23,6 @@ LABEL maintainer="${MAINTAINER}" ARG TARGETARCH -ENV ARCH=${TARGETARCH} - # skipcq: DOK-DL3008 RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -55,8 +53,8 @@ COPY versions versions COPY hack/go.mod.default hack/go.mod.default # basic deps -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make deps GO_CLEAN_DEPS=false \ && make ngt/install \ && make helm/install \ @@ -68,8 +66,8 @@ RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ && make kubectl/install # additional deps -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make k3d/install \ && make buf/install \ && make k9s/install \ diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile index d6819fe551..7de2fd243a 100644 --- a/dockers/discoverer/k8s/Dockerfile +++ b/dockers/discoverer/k8s/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -61,8 +60,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -81,8 +80,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile index 23dcafa1dd..2ca5eb6301 100644 --- a/dockers/gateway/filter/Dockerfile +++ b/dockers/gateway/filter/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -80,8 +79,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile index d54cd7b49d..7d67100c8c 100644 --- a/dockers/gateway/lb/Dockerfile +++ b/dockers/gateway/lb/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -80,8 +79,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile index d146917731..58d41d32f1 100644 --- a/dockers/gateway/mirror/Dockerfile +++ b/dockers/gateway/mirror/Dockerfile @@ -19,13 +19,12 @@ ARG DISTROLESS_IMAGE=gcr.io/distroless/static ARG DISTROLESS_IMAGE_TAG=nonroot ARG MAINTAINER="vdaas.org vald team " -FROM golang:${GO_VERSION} AS golang +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang -FROM ubuntu:devel AS builder +FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -56,8 +55,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ go mod download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -82,12 +81,12 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . COPY .git . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +FROM --platform=${BUILDPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} LABEL maintainer "${MAINTAINER}" ENV APP_NAME mirror diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile index bb7c3e3d3c..b230baa3e8 100644 --- a/dockers/index/job/correction/Dockerfile +++ b/dockers/index/job/correction/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -80,8 +79,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile index a0bd366138..aec6cded89 100644 --- a/dockers/index/job/creation/Dockerfile +++ b/dockers/index/job/creation/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -80,8 +79,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile index 36e19db46f..d22f7a6e95 100644 --- a/dockers/index/job/readreplica/rotate/Dockerfile +++ b/dockers/index/job/readreplica/rotate/Dockerfile @@ -20,13 +20,12 @@ ARG DISTROLESS_IMAGE=gcr.io/distroless/static ARG DISTROLESS_IMAGE_TAG=nonroot ARG MAINTAINER="vdaas.org vald team " -FROM golang:${GO_VERSION} AS golang +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang -FROM ubuntu:devel AS builder +FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -80,15 +79,15 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} RUN cp sample.yaml /tmp/config.yaml -FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +FROM --platform=${BUILDPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} LABEL maintainer="${MAINTAINER}" ENV APP_NAME readreplica-rotate diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile index 6b5a6beb39..9146892bba 100644 --- a/dockers/index/job/save/Dockerfile +++ b/dockers/index/job/save/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -80,8 +79,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile index 72a77f0cd7..056624f3ff 100644 --- a/dockers/manager/index/Dockerfile +++ b/dockers/manager/index/Dockerfile @@ -26,7 +26,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -80,8 +79,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index 3d678ee7ba..acef095c7e 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -23,16 +23,15 @@ ARG DISTROLESS_IMAGE_TAG=nonroot ARG UPX_OPTIONS=-9 ARG MAINTAINER="vdaas.org vald team " -FROM golang:${GO_VERSION} AS golang +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang -FROM ubuntu:devel AS builder +FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG UPX_OPTIONS ARG ZLIB_VERSION ARG HDF5_VERSION ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -90,8 +89,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -110,15 +109,15 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} RUN cp sample.yaml /tmp/config.yaml -FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +FROM --platform=${BUILDPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} LABEL maintainer="${MAINTAINER}" ENV APP_NAME job diff --git a/dockers/tools/benchmark/operator/Dockerfile b/dockers/tools/benchmark/operator/Dockerfile index 22e298b3f2..e629f68ca1 100644 --- a/dockers/tools/benchmark/operator/Dockerfile +++ b/dockers/tools/benchmark/operator/Dockerfile @@ -21,13 +21,12 @@ ARG DISTROLESS_IMAGE_TAG=nonroot ARG UPX_OPTIONS=-9 ARG MAINTAINER="vdaas.org vald team " -FROM golang:${GO_VERSION} AS golang +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang -FROM ubuntu:devel AS builder +FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -65,8 +64,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal @@ -85,15 +84,15 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions COPY versions . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} RUN cp sample.yaml /tmp/config.yaml -FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +FROM --platform=${BUILDPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} LABEL maintainer="${MAINTAINER}" ENV APP_NAME operator diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile index 22c181fc9c..eeabbc9dda 100644 --- a/dockers/tools/cli/loadtest/Dockerfile +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -24,7 +24,6 @@ FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder ARG TARGETARCH -ENV ARCH=${TARGETARCH} ENV GO111MODULE on ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -60,8 +59,8 @@ COPY .git . COPY go.mod . COPY go.sum . -RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \ +RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ make go/download WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal From 15ce01d67d6b5bd3e0aaab06a74d22b90022f9d7 Mon Sep 17 00:00:00 2001 From: Hiroto Funakoshi Date: Wed, 24 Jan 2024 15:10:45 +0900 Subject: [PATCH 03/10] Add internal KVS pogreb package (#2302) * feat: add internal pogreb kvs package Signed-off-by: hlts2 * fix: typo Signed-off-by: hlts2 * fix: tweak Signed-off-by: hlts2 * fix: deepsource warning Signed-off-by: hlts2 * feat: add test for kvs and small refactor Signed-off-by: hlts2 * feat: add condition to check key existance Signed-off-by: hlts2 * fix: small refactor Signed-off-by: hlts2 * fix: tuning option value Signed-off-by: hlts2 * fix: deepsource warning Signed-off-by: hlts2 * fix: make format Signed-off-by: hlts2 * Update internal/db/kvs/pogreb/pogreb.go Co-authored-by: datelier <57349093+datelier@users.noreply.github.com> * fix: error handling for option and update comment Signed-off-by: hlts2 * fix: option setting Signed-off-by: hlts2 --------- Signed-off-by: hlts2 Co-authored-by: datelier <57349093+datelier@users.noreply.github.com> --- go.mod | 1 + go.sum | 2 + hack/go.mod.default | 1 + internal/db/kvs/pogreb/options.go | 84 +++ internal/db/kvs/pogreb/options_test.go | 261 ++++++++ internal/db/kvs/pogreb/pogreb.go | 137 +++++ internal/db/kvs/pogreb/pogreb_test.go | 801 +++++++++++++++++++++++++ 7 files changed, 1287 insertions(+) create mode 100644 internal/db/kvs/pogreb/options.go create mode 100644 internal/db/kvs/pogreb/options_test.go create mode 100644 internal/db/kvs/pogreb/pogreb.go create mode 100644 internal/db/kvs/pogreb/pogreb_test.go diff --git a/go.mod b/go.mod index a066a0af80..17a2dfe2ef 100755 --- a/go.mod +++ b/go.mod @@ -421,6 +421,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect + github.com/akrylysov/pogreb v0.10.2 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/campoy/embedmd v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5e745c898a..bc9f46a40e 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyR github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= +github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/stroke v0.0.0-20221221101821-bd29b49d73f0/go.mod h1:ccdDYaY5+gO+cbnQdFxEXqfy0RkoV25H3jLXUDNM3wg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= diff --git a/hack/go.mod.default b/hack/go.mod.default index ba685da2ca..bd045ef535 100755 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -234,6 +234,7 @@ replace ( github.com/pkg/errors => github.com/pkg/errors upgrade github.com/pkg/sftp => github.com/pkg/sftp upgrade github.com/pmezard/go-difflib => github.com/pmezard/go-difflib upgrade + github.com/akrylysov/pogreb => github.com/akrylysov/pogreb upgrade github.com/prashantv/gostub => github.com/prashantv/gostub upgrade github.com/prometheus/client_golang => github.com/prometheus/client_golang upgrade github.com/prometheus/client_model => github.com/prometheus/client_model upgrade diff --git a/internal/db/kvs/pogreb/options.go b/internal/db/kvs/pogreb/options.go new file mode 100644 index 0000000000..58c62728a9 --- /dev/null +++ b/internal/db/kvs/pogreb/options.go @@ -0,0 +1,84 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package pogreb + +import ( + "time" + + "github.com/akrylysov/pogreb" +) + +var deafultOpts = []Option{ + WithPath("pogreb.db"), + WithBackgroundSyncInterval("5s"), +} + +// Option represents the functional option for database. +type Option func(*db) error + +// WithPath returns the option to set path. +func WithPath(path string) Option { + return func(d *db) error { + if path != "" { + d.path = path + } + return nil + } +} + +// WithBackgroundSyncInterval returns the option to sets the amount of time between background Sync() calls. +// Setting the value to 0 disables the automatic background synchronization. +// Setting the value to -1 or less makes the DB call Sync() after every write operation. +func WithBackgroundSyncInterval(s string) Option { + return func(d *db) error { + if s == "" { + return nil + } + dur, err := time.ParseDuration(s) + if err != nil { + return err + } + if d.opts == nil { + d.opts = new(pogreb.Options) + } + if dur < -1 { + dur = -1 + } + d.opts.BackgroundSyncInterval = dur + return nil + } +} + +// WithBackgroundCompactionInterval returns the option to sets the amount of time between background Compact() calls. +// Setting the value to 0 or less disables the automatic background compaction. +func WithBackgroundCompactionInterval(s string) Option { + return func(d *db) error { + if s == "" { + return nil + } + dur, err := time.ParseDuration(s) + if err != nil { + return err + } + if d.opts == nil { + d.opts = new(pogreb.Options) + } + + if dur < 0 { + dur = 0 + } + d.opts.BackgroundCompactionInterval = dur + return nil + } +} diff --git a/internal/db/kvs/pogreb/options_test.go b/internal/db/kvs/pogreb/options_test.go new file mode 100644 index 0000000000..58d0abf847 --- /dev/null +++ b/internal/db/kvs/pogreb/options_test.go @@ -0,0 +1,261 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package pogreb + +import ( + "reflect" + "testing" + "time" + + "github.com/akrylysov/pogreb" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/test/goleak" +) + +func TestWithPath(t *testing.T) { + t.Parallel() + type args struct { + path string + } + type want struct { + want Option + err error + path string + } + type test struct { + name string + args args + want want + checkFunc func(want, *db, error) error + beforeFunc func(*testing.T, args) + afterFunc func(*testing.T, args) + } + defaultCheckFunc := func(w want, got *db, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if err == nil { + if !reflect.DeepEqual(got.path, w.path) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + } + return nil + } + tests := []test{ + func() test { + path := "db-dir" + return test{ + name: "Succeeds to apply option", + args: args{ + path: path, + }, + want: want{ + path: path, + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(tt, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := new(db) + err := WithPath(test.args.path)(got) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func TestWithBackgroundSyncInterval(t *testing.T) { + t.Parallel() + type args struct { + s string + } + type want struct { + want Option + err error + opts *pogreb.Options + } + type test struct { + name string + args args + want want + checkFunc func(want, *db, error) error + beforeFunc func(*testing.T, args) + afterFunc func(*testing.T, args) + } + defaultCheckFunc := func(w want, got *db, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if err == nil { + if !reflect.DeepEqual(got.opts.BackgroundSyncInterval, w.opts.BackgroundSyncInterval) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + } + return nil + } + tests := []test{ + func() test { + dur := "100ms" + return test{ + name: "Succeeds to apply option", + args: args{ + s: dur, + }, + want: want{ + opts: &pogreb.Options{ + BackgroundSyncInterval: 100 * time.Millisecond, + }, + }, + } + }(), + func() test { + dur := "invalid" + return test{ + name: "Fails to apply option with invalid value", + args: args{ + s: dur, + }, + want: want{ + err: errors.New("time: invalid duration \"invalid\""), + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(tt, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := new(db) + err := WithBackgroundSyncInterval(test.args.s)(got) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func TestWithBackgroundCompactionInterval(t *testing.T) { + t.Parallel() + type args struct { + s string + } + type want struct { + want Option + err error + opts *pogreb.Options + } + type test struct { + name string + args args + want want + checkFunc func(want, *db, error) error + beforeFunc func(*testing.T, args) + afterFunc func(*testing.T, args) + } + defaultCheckFunc := func(w want, got *db, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if err == nil { + if !reflect.DeepEqual(got.opts.BackgroundCompactionInterval, w.opts.BackgroundCompactionInterval) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + } + return nil + } + tests := []test{ + func() test { + dur := "100ms" + return test{ + name: "Succeeds to apply option", + args: args{ + s: dur, + }, + want: want{ + opts: &pogreb.Options{ + BackgroundCompactionInterval: 100 * time.Millisecond, + }, + }, + } + }(), + func() test { + dur := "invalid" + return test{ + name: "Fails to apply option with invalid value", + args: args{ + s: dur, + }, + want: want{ + err: errors.New("time: invalid duration \"invalid\""), + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(tt, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := new(db) + err := WithBackgroundCompactionInterval(test.args.s)(got) + if err := checkFunc(test.want, got, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +// NOT IMPLEMENTED BELOW diff --git a/internal/db/kvs/pogreb/pogreb.go b/internal/db/kvs/pogreb/pogreb.go new file mode 100644 index 0000000000..d2dc8854bd --- /dev/null +++ b/internal/db/kvs/pogreb/pogreb.go @@ -0,0 +1,137 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package pogreb + +import ( + "context" + "os" + "reflect" + + "github.com/akrylysov/pogreb" + + "github.com/vdaas/vald/internal/conv" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" +) + +// Pogreb represents an interface for operating the pogreb database. +type Pogreb interface { + Set(key string, val []byte) error + Get(key string) ([]byte, bool, error) + Delete(key string) error + Range(ctx context.Context, f func(key string, val []byte) bool) error + Len() uint32 + Close(remove bool) error +} + +type db struct { + db *pogreb.DB + opts *pogreb.Options + path string +} + +// New returns a new pogreb instance. +// If the directory path does not exist, it creates a directory for database. +// If opts is nil, it uses default options. +func New(opts ...Option) (_ Pogreb, err error) { + db := new(db) + for _, opt := range append(deafultOpts, opts...) { + if err := opt(db); err != nil { + oerr := errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + e := &errors.ErrCriticalOption{} + if errors.As(oerr, &e) { + log.Error(err) + return nil, oerr + } + log.Warn(oerr) + } + } + + // If db.opts is nil, a default value is used. + db.db, err = pogreb.Open(db.path, db.opts) + if err != nil { + return nil, err + } + return db, nil +} + +// Set sets the value for a key. +func (d *db) Set(key string, val []byte) error { + return d.db.Put(conv.Atob(key), val) +} + +// Get returns the value stored in the database for a key. +// The ok result indicates whether value was found in the database. +func (d *db) Get(key string) ([]byte, bool, error) { + val, err := d.db.Get(conv.Atob(key)) + if err != nil { + return nil, false, err + } + // If val is nil, it means that there is no value associated with key, so false is returned. + if val == nil { + return nil, false, nil + } + return val, true, nil +} + +// Delete deletes the given key from the database. +func (d *db) Delete(key string) error { + // NOTE: Even if the key does not exist in the database, no error occurs. + // Depending on the future use case, it may be necessary to check for the existence of the key, in which case the `Has` method can be used. + return d.db.Delete(conv.Atob(key)) +} + +// Range calls f sequentially for each key and value present in the database. +// If f returns false, range stops the iteration. +func (d *db) Range(ctx context.Context, f func(key string, val []byte) bool) error { + it := d.db.Items() + for { + select { + case <-ctx.Done(): + return nil + default: + key, val, err := it.Next() + if err != nil { + if errors.Is(err, pogreb.ErrIterationDone) { + return nil + } + return err + } + if !f(conv.Btoa(key), val) { + return nil + } + } + } +} + +// Len returns the number of keys in the DB. +func (d *db) Len() uint32 { + return d.db.Count() +} + +// Close closes the database and removes the file if remove is true. +func (d *db) Close(remove bool) (err error) { + if serr := d.db.Sync(); serr != nil { + err = serr + } + if cerr := d.db.Close(); cerr != nil { + err = errors.Join(err, cerr) + } + if remove { + if rerr := os.RemoveAll(d.path); rerr != nil { + err = errors.Join(err, rerr) + } + } + return err +} diff --git a/internal/db/kvs/pogreb/pogreb_test.go b/internal/db/kvs/pogreb/pogreb_test.go new file mode 100644 index 0000000000..ac4dda8a82 --- /dev/null +++ b/internal/db/kvs/pogreb/pogreb_test.go @@ -0,0 +1,801 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package pogreb + +import ( + "context" + "reflect" + "testing" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/test/goleak" +) + +func TestNew(t *testing.T) { + t.Parallel() + type args struct { + opts []Option + } + type want struct { + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, error) error + beforeFunc func(*testing.T, args) + afterFunc func(*testing.T, args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + return nil + } + tests := []test{ + func() test { + return test{ + name: "Succeeds to create pogreb instance with new path when the path does not exist", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + }, + } + }(), + func() test { + opts := []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + } + return test{ + name: " Succeeds to restart the pogres instance", + args: args{ + opts: opts, + }, + beforeFunc: func(t *testing.T, _ args) { + t.Helper() + db, err := New(opts...) + if err != nil { + t.Fatal(err) + } + if err := db.Close(false); err != nil { + t.Fatal(err) + } + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(tt, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + _, err := New(test.args.opts...) + if err := checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +// NOTE: To test Get, data needs to be inserted beforehand, so we test Set and Get together. +// If the test function name is changed, it is regenerated by gotests, so the function name is kept the same. +func Test_db_Get(t *testing.T) { + t.Parallel() + type args struct { + opts []Option + key string + } + type want struct { + want []byte + want1 bool + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, []byte, bool, error) error + beforeFunc func(*testing.T, Pogreb, args) + afterFunc func(*testing.T, Pogreb, args) + } + defaultCheckFunc := func(w want, got []byte, got1 bool, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + if !reflect.DeepEqual(got1, w.want1) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got1, w.want1) + } + return nil + } + tests := []test{ + func() test { + var ( + key = "key" + val = []byte("val") + ) + return test{ + name: "Succeeds to get the value associated with the key", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + key: key, + }, + want: want{ + want: val, + want1: true, + }, + beforeFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Set(key, val); err != nil { + t.Fatal(err) + } + }, + afterFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Close(true); err != nil { + t.Fatal(err) + } + }, + } + }(), + func() test { + var ( + key = "key" + val = []byte("val") + ) + return test{ + name: "Fails to get the value associated with the key if it does not exist", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + key: "not-exist", + }, + want: want{ + want1: false, + }, + beforeFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Set(key, val); err != nil { + t.Fatal(err) + } + }, + afterFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Close(true); err != nil { + t.Fatal(err) + } + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + + d, err := New(test.args.opts...) + if err != nil { + t.Fatal(err) + } + if test.beforeFunc != nil { + test.beforeFunc(tt, d, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, d, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got, got1, err := d.Get(test.args.key) + if err := checkFunc(test.want, got, got1, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +// NOTE: To test Delete, data needs to be inserted beforehand, so we test Set and Delete together. +// If the test function name is changed, it is regenerated by gotests, so the function name is kept the same. +func Test_db_Delete(t *testing.T) { + t.Parallel() + type args struct { + opts []Option + key string + } + type want struct { + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, Pogreb, error) error + beforeFunc func(*testing.T, Pogreb, args) + afterFunc func(*testing.T, Pogreb, args) + } + defaultCheckFunc := func(w want, _ Pogreb, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + return nil + } + tests := []test{ + func() test { + var ( + key = "key" + val = []byte("val") + ) + return test{ + name: "Succeeds to delete the value associated with the key", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + key: key, + }, + checkFunc: func(w want, d Pogreb, err error) error { + if err := defaultCheckFunc(w, d, err); err != nil { + return err + } + _, ok, err := d.Get(key) + if err != nil { + return err + } + if ok { + return errors.New("key exists") + } + return nil + }, + beforeFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Set(key, val); err != nil { + t.Fatal(err) + } + }, + afterFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Close(true); err != nil { + t.Fatal(err) + } + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + + d, err := New(test.args.opts...) + if err != nil { + t.Fatal(err) + } + if test.beforeFunc != nil { + test.beforeFunc(tt, d, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, d, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + err = d.Delete(test.args.key) + if err := checkFunc(test.want, d, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +// skipcq: GO-R1005 +func Test_db_Range(t *testing.T) { + t.Parallel() + type args struct { + opts []Option + ctx context.Context + f func(key string, val []byte) bool + } + type want struct { + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, error) error + beforeFunc func(*testing.T, Pogreb, args) + afterFunc func(*testing.T, Pogreb, args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + return nil + } + tests := []test{ + func() test { + data := map[string][]byte{ + "key-1": []byte("val-1"), + "key-2": []byte("val-2"), + } + got := make(map[string][]byte) + return test{ + name: "Succeeds to get all keys", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + ctx: context.Background(), + f: func(key string, val []byte) bool { + got[key] = val + return true + }, + }, + checkFunc: func(w want, err error) error { + if err := defaultCheckFunc(w, err); err != nil { + return err + } + if !reflect.DeepEqual(got, data) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, data) + } + return nil + }, + beforeFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + for key, val := range data { + if err := d.Set(key, val); err != nil { + t.Fatal(err) + } + } + }, + afterFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Close(true); err != nil { + t.Fatal(err) + } + }, + } + }(), + func() test { + got := make(map[string][]byte) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return test{ + name: "Fails to get all keys when the context is canceled", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + ctx: ctx, + f: func(key string, val []byte) bool { + got[key] = val + return true + }, + }, + checkFunc: func(w want, err error) error { + if err := defaultCheckFunc(w, err); err != nil { + return err + } + if data := make(map[string][]byte); !reflect.DeepEqual(got, data) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, data) + } + return nil + }, + beforeFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + data := map[string][]byte{ + "key-1": []byte("val-1"), + "key-2": []byte("val-2"), + } + for key, val := range data { + if err := d.Set(key, val); err != nil { + t.Fatal(err) + } + } + }, + afterFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Close(true); err != nil { + t.Fatal(err) + } + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + + d, err := New(test.args.opts...) + if err != nil { + t.Fatal(err) + } + if test.beforeFunc != nil { + test.beforeFunc(tt, d, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, d, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + err = d.Range(test.args.ctx, test.args.f) + if err := checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func Test_db_Len(t *testing.T) { + t.Parallel() + type args struct { + opts []Option + } + type want struct { + want uint32 + } + type test struct { + name string + args args + want want + checkFunc func(want, uint32) error + beforeFunc func(*testing.T, Pogreb, args) + afterFunc func(*testing.T, Pogreb, args) + } + defaultCheckFunc := func(w want, got uint32) error { + if !reflect.DeepEqual(got, w.want) { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) + } + return nil + } + tests := []test{ + func() test { + data := map[string][]byte{ + "key-1": []byte("val-1"), + "key-2": []byte("val-2"), + } + return test{ + name: "Succeeds to get the number of keys", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + }, + want: want{ + want: uint32(len(data)), + }, + beforeFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + for key, val := range data { + if err := d.Set(key, val); err != nil { + t.Fatal(err) + } + } + }, + afterFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Close(true); err != nil { + t.Fatal(err) + } + }, + } + }(), + func() test { + return test{ + name: "Succeeds to get the number of keys when key does not exist", + args: args{ + opts: []Option{ + WithPath(t.TempDir()), + WithBackgroundSyncInterval("0s"), + }, + }, + afterFunc: func(t *testing.T, d Pogreb, args args) { + t.Helper() + if err := d.Close(true); err != nil { + t.Fatal(err) + } + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + + d, err := New(test.args.opts...) + if err != nil { + t.Fatal(err) + } + if test.beforeFunc != nil { + test.beforeFunc(tt, d, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, d, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + + got := d.Len() + if err := checkFunc(test.want, got); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +// NOT IMPLEMENTED BELOW +// +// +// func Test_db_Set(t *testing.T) { +// type args struct { +// key string +// val []byte +// } +// type fields struct { +// db *pogreb.DB +// opts *pogreb.Options +// path string +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// key:"", +// val:nil, +// }, +// fields: fields { +// db:nil, +// opts:nil, +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// key:"", +// val:nil, +// }, +// fields: fields { +// db:nil, +// opts:nil, +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// d := &db{ +// db: test.fields.db, +// opts: test.fields.opts, +// path: test.fields.path, +// } +// +// err := d.Set(test.args.key, test.args.val) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_db_Close(t *testing.T) { +// type args struct { +// remove bool +// } +// type fields struct { +// db *pogreb.DB +// opts *pogreb.Options +// path string +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// remove:false, +// }, +// fields: fields { +// db:nil, +// opts:nil, +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// remove:false, +// }, +// fields: fields { +// db:nil, +// opts:nil, +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// d := &db{ +// db: test.fields.db, +// opts: test.fields.opts, +// path: test.fields.path, +// } +// +// err := d.Close(test.args.remove) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } From a037bd39b121de79e029011407992d5267edc918 Mon Sep 17 00:00:00 2001 From: Kiichiro YUKAWA Date: Mon, 29 Jan 2024 09:38:46 +0900 Subject: [PATCH 04/10] :green_heart: Add dispatch workflow for update contents of vdaas/web repo (#2294) Signed-off-by: vankichi --- .github/workflows/update-web-docs.yml | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/update-web-docs.yml diff --git a/.github/workflows/update-web-docs.yml b/.github/workflows/update-web-docs.yml new file mode 100644 index 0000000000..442ae85cf0 --- /dev/null +++ b/.github/workflows/update-web-docs.yml @@ -0,0 +1,43 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Update web contents" +on: + workflow_dispatch: + push: + branches: + - main + - "release/v*.*" + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - "**.md" + - "assets/docs/**" +jobs: + dispatch: + runs-on: ubuntu-latest + steps: + - name: Dispatch + run: | + curl --fail -u "${USER}:${TOKEN}" \ + -X POST https://api.github.com/repos/vdaas/web/dispatches \ + -H 'Accept: application/vnd.github.everest-preview+json' \ + --data '{"event_type": "update-contents"}' + env: + USER: ${{ secrets.DISPATCH_USER }} + TOKEN: ${{ secrets.DISPATCH_TOKEN }} From 183f037900e0111aafbf423b1a3a44d75a1276ee Mon Sep 17 00:00:00 2001 From: Yusuke Kadowaki Date: Mon, 29 Jan 2024 14:54:34 +0900 Subject: [PATCH 05/10] Add issue metrics (#2308) --- .github/workflows/issue-metrics.yaml | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/issue-metrics.yaml diff --git a/.github/workflows/issue-metrics.yaml b/.github/workflows/issue-metrics.yaml new file mode 100644 index 0000000000..d79a13bf59 --- /dev/null +++ b/.github/workflows/issue-metrics.yaml @@ -0,0 +1,48 @@ +name: Monthly review time metrics +on: + workflow_dispatch: + inputs: + time_window: + required: false + description: 'Time window for the issue metrics report. e.g. 2021-01-01..2021-01-31. If not set, the previous month will be calculated.' + type: string + schedule: + - cron: '3 2 1 * *' +permissions: + issues: write + pull-requests: read +jobs: + build: + name: review time metrics + runs-on: ubuntu-latest + steps: + - name: Get dates for last month + shell: bash + run: | + # If TIME_WINDOW is set, use it as last_month + if [ -n "$TIME_WINDOW" ]; then + echo "last_month=$TIME_WINDOW" >> "$GITHUB_ENV" + else + # Calculate the first day of the previous month + first_day=$(date -d "last month" +%Y-%m-01) + + # Calculate the last day of the previous month + last_day=$(date -d "$first_day +1 month -1 day" +%Y-%m-%d) + + # Set an environment variable with the date range + echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV" + fi + env: + TIME_WINDOW: ${{ github.event.inputs.time_window }} + - name: Run issue-metrics tool + uses: github/issue-metrics@v2 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SEARCH_QUERY: 'repo:vdaas/vald is:pr created:${{ env.last_month }}' + HIDE_TIME_TO_ANSWER: true + - name: Create issue + uses: peter-evans/create-issue-from-file@v4 + with: + title: "Monthly review time metrics report: ${{ env.last_month }}" + token: ${{ secrets.GITHUB_TOKEN }} + content-filepath: ./issue_metrics.md From 73416fcece67ac00f083cd801754ee53fc0dc915 Mon Sep 17 00:00:00 2001 From: Yusuke Kadowaki Date: Mon, 29 Jan 2024 16:57:34 +0900 Subject: [PATCH 06/10] Add HPA for read replica (#2307) * Add hpa for read replica * Update schema * Add read replica flag to agent ngt to skip save index when it's read replica instance * Refactor * Add tests for Close * enable readreplica hpa on e2e * Add error handling for write operation to read replica in CreateIndex and saveIndex --- .github/helm/values/values-readreplica.yaml | 3 +- charts/vald-benchmark-operator/README.md | 186 ++++++++++ .../vald-helm-operator/crds/valdrelease.yaml | 190 +++++++++- .../vald-readreplica/templates/configmap.yaml | 47 +++ .../templates/deployment.yaml | 8 +- charts/vald-readreplica/templates/hpa.yaml | 67 ++++ charts/vald/README.md | 97 ++++- .../vald/templates/gateway/lb/configmap.yaml | 4 +- charts/vald/values.schema.json | 340 +++++++++++++++++- charts/vald/values.yaml | 20 +- internal/config/ngt.go | 3 + internal/errors/agent.go | 3 + pkg/agent/core/ngt/service/ngt.go | 22 +- pkg/agent/core/ngt/service/ngt_test.go | 205 +++++++++++ pkg/agent/core/ngt/service/option.go | 8 + pkg/agent/core/ngt/usecase/agentd.go | 1 + 16 files changed, 1182 insertions(+), 22 deletions(-) create mode 100644 charts/vald-benchmark-operator/README.md create mode 100644 charts/vald-readreplica/templates/configmap.yaml create mode 100644 charts/vald-readreplica/templates/hpa.yaml diff --git a/.github/helm/values/values-readreplica.yaml b/.github/helm/values/values-readreplica.yaml index 2e4b86b89f..6f392b8816 100644 --- a/.github/helm/values/values-readreplica.yaml +++ b/.github/helm/values/values-readreplica.yaml @@ -58,7 +58,8 @@ agent: readreplica: enabled: true snapshot_classname: "csi-hostpath-snapclass" - replica: 1 + hpa: + enabled: true discoverer: minReplicas: 1 diff --git a/charts/vald-benchmark-operator/README.md b/charts/vald-benchmark-operator/README.md new file mode 100644 index 0000000000..77a8b21770 --- /dev/null +++ b/charts/vald-benchmark-operator/README.md @@ -0,0 +1,186 @@ +# vald-benchmark-operator + +![Version: v1.7.5](https://img.shields.io/badge/Version-v1.7.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) + +A benchmark operator for benchmarking the Vald cluster. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| -------- | -------------------- | --- | +| kpango | | | +| vankichi | | | +| kmrmt | | | + +## Source Code + +- + +## Values + +| Key | Type | Default | Description | +| ----------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| affinity | object | `{}` | affinity | +| annotations | object | `{}` | deployment annotations | +| image.pullPolicy | string | `"Always"` | image pull policy | +| image.repository | string | `"vdaas/vald-benchmark-operator"` | job image repository | +| image.tag | string | `"v1.7.5"` | image tag for job docker image | +| job_image.pullPolicy | string | `"Always"` | | +| job_image.repository | string | `"vdaas/vald-benchmark-job"` | | +| job_image.tag | string | `"v1.7.5"` | | +| logging.format | string | `"raw"` | logging format. logging format must be `raw` or `json` | +| logging.level | string | `"debug"` | logging level. logging level must be `debug`, `info`, `warn`, `error` or `fatal`. | +| logging.logger | string | `"glg"` | logger name. currently logger must be `glg` or `zap`. | +| name | string | `"vald-benchmark-operator"` | name of the deployment | +| nodeSelector | object | `{}` | node labels for pod assignment | +| observability.enabled | bool | `false` | | +| observability.metrics.enable_cgo | bool | `true` | | +| observability.metrics.enable_goroutine | bool | `true` | | +| observability.metrics.enable_memory | bool | `true` | | +| observability.metrics.enable_version_info | bool | `true` | | +| observability.metrics.version_info_labels[0] | string | `"vald_version"` | | +| observability.metrics.version_info_labels[1] | string | `"server_name"` | | +| observability.metrics.version_info_labels[2] | string | `"git_commit"` | | +| observability.metrics.version_info_labels[3] | string | `"build_time"` | | +| observability.metrics.version_info_labels[4] | string | `"go_version"` | | +| observability.metrics.version_info_labels[5] | string | `"go_os"` | | +| observability.metrics.version_info_labels[6] | string | `"go_arch"` | | +| observability.metrics.version_info_labels[7] | string | `"ngt_version"` | | +| observability.otlp.attribute.namespace | string | `"_MY_POD_NAMESPACE_"` | | +| observability.otlp.attribute.node_name | string | `"_MY_NODE_NAME_"` | | +| observability.otlp.attribute.pod_name | string | `"_MY_POD_NAME_"` | | +| observability.otlp.attribute.service_name | string | `"vald-benchmark-operator"` | | +| observability.otlp.collector_endpoint | string | `""` | | +| observability.otlp.metrics_export_interval | string | `"1s"` | | +| observability.otlp.metrics_export_timeout | string | `"1m"` | | +| observability.otlp.trace_batch_timeout | string | `"1s"` | | +| observability.otlp.trace_export_timeout | string | `"1m"` | | +| observability.otlp.trace_max_export_batch_size | int | `1024` | | +| observability.otlp.trace_max_queue_size | int | `256` | | +| observability.trace.enabled | bool | `false` | | +| observability.trace.sampling_rate | int | `1` | | +| podAnnotations | object | `{}` | pod annotations | +| podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | +| rbac.create | bool | `true` | required roles and rolebindings will be created | +| rbac.name | string | `"vald-benchmark-operator"` | name of roles and rolebindings | +| replicas | int | `1` | the number of replica for deployment | +| resources | object | `{"limits":{"cpu":"300m","memory":"300Mi"},"requests":{"cpu":"200m","memory":"200Mi"}}` | kubernetes resources of pod | +| securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for container | +| server_config.full_shutdown_duration | string | `"600s"` | | +| server_config.healths.liveness.enabled | bool | `true` | | +| server_config.healths.liveness.host | string | `"0.0.0.0"` | | +| server_config.healths.liveness.livenessProbe.failureThreshold | int | `2` | liveness probe failure threshold | +| server_config.healths.liveness.livenessProbe.httpGet.path | string | `"/liveness"` | readiness probe path | +| server_config.healths.liveness.livenessProbe.httpGet.port | string | `"liveness"` | readiness probe port | +| server_config.healths.liveness.livenessProbe.httpGet.scheme | string | `"HTTP"` | readiness probe scheme | +| server_config.healths.liveness.livenessProbe.initialDelaySeconds | int | `15` | liveness probe initial delay seconds | +| server_config.healths.liveness.livenessProbe.periodSeconds | int | `20` | liveness probe period seconds | +| server_config.healths.liveness.livenessProbe.successThreshold | int | `1` | liveness probe success threshold | +| server_config.healths.liveness.livenessProbe.timeoutSeconds | int | `5` | liveness probe timeout seconds | +| server_config.healths.liveness.port | int | `3000` | | +| server_config.healths.liveness.server.http.handler_timeout | string | `""` | | +| server_config.healths.liveness.server.http.idle_timeout | string | `""` | | +| server_config.healths.liveness.server.http.read_header_timeout | string | `""` | | +| server_config.healths.liveness.server.http.read_timeout | string | `""` | | +| server_config.healths.liveness.server.http.shutdown_duration | string | `"5s"` | | +| server_config.healths.liveness.server.http.write_timeout | string | `""` | | +| server_config.healths.liveness.server.mode | string | `""` | | +| server_config.healths.liveness.server.network | string | `"tcp"` | | +| server_config.healths.liveness.server.probe_wait_time | string | `"3s"` | | +| server_config.healths.liveness.server.socket_path | string | `""` | | +| server_config.healths.liveness.servicePort | int | `3000` | | +| server_config.healths.readiness.enabled | bool | `true` | | +| server_config.healths.readiness.host | string | `"0.0.0.0"` | | +| server_config.healths.readiness.port | int | `3001` | | +| server_config.healths.readiness.readinessProbe.failureThreshold | int | `2` | readiness probe failure threshold | +| server_config.healths.readiness.readinessProbe.httpGet.path | string | `"/readiness"` | readiness probe path | +| server_config.healths.readiness.readinessProbe.httpGet.port | string | `"readiness"` | readiness probe port | +| server_config.healths.readiness.readinessProbe.httpGet.scheme | string | `"HTTP"` | readiness probe scheme | +| server_config.healths.readiness.readinessProbe.initialDelaySeconds | int | `10` | readiness probe initial delay seconds | +| server_config.healths.readiness.readinessProbe.periodSeconds | int | `3` | readiness probe period seconds | +| server_config.healths.readiness.readinessProbe.successThreshold | int | `1` | readiness probe success threshold | +| server_config.healths.readiness.readinessProbe.timeoutSeconds | int | `2` | readiness probe timeout seconds | +| server_config.healths.readiness.server.http.handler_timeout | string | `""` | | +| server_config.healths.readiness.server.http.idle_timeout | string | `""` | | +| server_config.healths.readiness.server.http.read_header_timeout | string | `""` | | +| server_config.healths.readiness.server.http.read_timeout | string | `""` | | +| server_config.healths.readiness.server.http.shutdown_duration | string | `"0s"` | | +| server_config.healths.readiness.server.http.write_timeout | string | `""` | | +| server_config.healths.readiness.server.mode | string | `""` | | +| server_config.healths.readiness.server.network | string | `"tcp"` | | +| server_config.healths.readiness.server.probe_wait_time | string | `"3s"` | | +| server_config.healths.readiness.server.socket_path | string | `""` | | +| server_config.healths.readiness.servicePort | int | `3001` | | +| server_config.healths.startup.enabled | bool | `true` | enable startup probe. | +| server_config.healths.startup.startupProbe.failureThreshold | int | `30` | | +| server_config.healths.startup.startupProbe.httpGet.path | string | `"/liveness"` | | +| server_config.healths.startup.startupProbe.httpGet.port | string | `"liveness"` | | +| server_config.healths.startup.startupProbe.httpGet.scheme | string | `"HTTP"` | | +| server_config.healths.startup.startupProbe.initialDelaySeconds | int | `5` | | +| server_config.healths.startup.startupProbe.periodSeconds | int | `5` | | +| server_config.healths.startup.startupProbe.successThreshold | int | `1` | | +| server_config.healths.startup.startupProbe.timeoutSeconds | int | `2` | | +| server_config.metrics.pprof.enabled | bool | `false` | | +| server_config.metrics.pprof.host | string | `"0.0.0.0"` | | +| server_config.metrics.pprof.port | int | `6060` | | +| server_config.metrics.pprof.server.http.handler_timeout | string | `"5s"` | | +| server_config.metrics.pprof.server.http.idle_timeout | string | `"2s"` | | +| server_config.metrics.pprof.server.http.read_header_timeout | string | `"1s"` | | +| server_config.metrics.pprof.server.http.read_timeout | string | `"1s"` | | +| server_config.metrics.pprof.server.http.shutdown_duration | string | `"5s"` | | +| server_config.metrics.pprof.server.http.write_timeout | string | `"1m"` | | +| server_config.metrics.pprof.server.mode | string | `"REST"` | | +| server_config.metrics.pprof.server.network | string | `"tcp"` | | +| server_config.metrics.pprof.server.probe_wait_time | string | `"3s"` | | +| server_config.metrics.pprof.server.socket_path | string | `""` | | +| server_config.servers.grpc.enabled | bool | `true` | | +| server_config.servers.grpc.host | string | `"0.0.0.0"` | | +| server_config.servers.grpc.name | string | `"grpc"` | | +| server_config.servers.grpc.port | int | `8081` | | +| server_config.servers.grpc.server.grpc.bidirectional_stream_concurrency | int | `20` | | +| server_config.servers.grpc.server.grpc.connection_timeout | string | `""` | | +| server_config.servers.grpc.server.grpc.enable_reflection | bool | `true` | | +| server_config.servers.grpc.server.grpc.header_table_size | int | `0` | | +| server_config.servers.grpc.server.grpc.initial_conn_window_size | int | `0` | | +| server_config.servers.grpc.server.grpc.initial_window_size | int | `0` | | +| server_config.servers.grpc.server.grpc.interceptors | list | `[]` | | +| server_config.servers.grpc.server.grpc.keepalive.max_conn_age | string | `""` | gRPC server keep alive max connection age | +| server_config.servers.grpc.server.grpc.keepalive.max_conn_age_grace | string | `""` | gRPC server keep alive max connection age grace | +| server_config.servers.grpc.server.grpc.keepalive.max_conn_idle | string | `""` | gRPC server keep alive max connection idle | +| server_config.servers.grpc.server.grpc.keepalive.min_time | string | `"60s"` | gRPC server keep alive min_time | +| server_config.servers.grpc.server.grpc.keepalive.permit_without_stream | bool | `true` | gRPC server keep alive permit_without_stream | +| server_config.servers.grpc.server.grpc.keepalive.time | string | `"120s"` | gRPC server keep alive time | +| server_config.servers.grpc.server.grpc.keepalive.timeout | string | `"30s"` | gRPC server keep alive timeout | +| server_config.servers.grpc.server.grpc.max_header_list_size | int | `0` | | +| server_config.servers.grpc.server.grpc.max_receive_message_size | int | `0` | | +| server_config.servers.grpc.server.grpc.max_send_message_size | int | `0` | | +| server_config.servers.grpc.server.grpc.read_buffer_size | int | `0` | | +| server_config.servers.grpc.server.grpc.write_buffer_size | int | `0` | | +| server_config.servers.grpc.server.mode | string | `"GRPC"` | | +| server_config.servers.grpc.server.network | string | `"tcp"` | | +| server_config.servers.grpc.server.probe_wait_time | string | `"3s"` | | +| server_config.servers.grpc.server.restart | bool | `true` | | +| server_config.servers.grpc.server.socket_path | string | `""` | | +| server_config.servers.grpc.serviecPort | int | `8081` | | +| server_config.servers.rest.enabled | bool | `false` | | +| server_config.tls.ca | string | `"/path/to/ca"` | | +| server_config.tls.cert | string | `"/path/to/cert"` | | +| server_config.tls.enabled | bool | `false` | | +| server_config.tls.insecure_skip_verify | bool | `false` | enable/disable skip SSL certificate verification | +| server_config.tls.key | string | `"/path/to/key"` | | +| service.annotations | object | `{}` | service annotations | +| service.enabled | bool | `true` | service enabled | +| service.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| service.labels | object | `{}` | service labels | +| service.type | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| serviceAccount.create | bool | `true` | service account will be created | +| serviceAccount.name | string | `"vald-benchmark-operator"` | name of service account | +| time_zone | string | `""` | time_zone | +| tolerations | list | `[]` | tolerations | +| version | string | `"v0.0.0"` | version of benchmark-operator config | + +--- + +Autogenerated from chart metadata using [helm-docs v1.11.3](https://github.com/norwoodj/helm-docs/releases/v1.11.3) diff --git a/charts/vald-helm-operator/crds/valdrelease.yaml b/charts/vald-helm-operator/crds/valdrelease.yaml index 214ee2b95d..4a9345982c 100644 --- a/charts/vald-helm-operator/crds/valdrelease.yaml +++ b/charts/vald-helm-operator/crds/valdrelease.yaml @@ -392,12 +392,23 @@ spec: type: string enabled: type: boolean + hpa: + type: object + properties: + enabled: + type: boolean + targetCPUUtilizationPercentage: + type: integer label_key: type: string + maxReplicas: + type: integer + minimum: 1 + minReplicas: + type: integer + minimum: 1 name: type: string - replica: - type: integer service: type: object properties: @@ -5314,6 +5325,181 @@ spec: type: boolean duration: type: string + read_client: + type: object + properties: + addrs: + type: array + items: + type: string + backoff: + type: object + properties: + backoff_factor: + type: number + backoff_time_limit: + type: string + enable_error_log: + type: boolean + initial_duration: + type: string + jitter_limit: + type: string + maximum_duration: + type: string + retry_count: + type: integer + call_option: + type: object + x-kubernetes-preserve-unknown-fields: true + circuit_breaker: + type: object + properties: + closed_error_rate: + type: number + closed_refresh_timeout: + type: string + half_open_error_rate: + type: number + min_samples: + type: integer + open_timeout: + type: string + connection_pool: + type: object + properties: + enable_dns_resolver: + type: boolean + enable_rebalance: + type: boolean + old_conn_close_duration: + type: string + rebalance_duration: + type: string + size: + type: integer + dial_option: + type: object + properties: + backoff_base_delay: + type: string + backoff_jitter: + type: number + backoff_max_delay: + type: string + backoff_multiplier: + type: number + enable_backoff: + type: boolean + initial_connection_window_size: + type: integer + initial_window_size: + type: integer + insecure: + type: boolean + interceptors: + type: array + items: + type: string + enum: + - TraceInterceptor + keepalive: + type: object + properties: + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_msg_size: + type: integer + min_connection_timeout: + type: string + net: + type: object + properties: + dialer: + type: object + properties: + dual_stack_enabled: + type: boolean + keepalive: + type: string + timeout: + type: string + dns: + type: object + properties: + cache_enabled: + type: boolean + cache_expiration: + type: string + refresh_duration: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + read_buffer_size: + type: integer + timeout: + type: string + write_buffer_size: + type: integer + health_check_duration: + type: string + max_recv_msg_size: + type: integer + max_retry_rpc_buffer_size: + type: integer + max_send_msg_size: + type: integer + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + wait_for_ready: + type: boolean index_replica: type: integer minimum: 1 diff --git a/charts/vald-readreplica/templates/configmap.yaml b/charts/vald-readreplica/templates/configmap.yaml new file mode 100644 index 0000000000..f928c96752 --- /dev/null +++ b/charts/vald-readreplica/templates/configmap.yaml @@ -0,0 +1,47 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $agent := .Values.agent -}} +{{- $readreplica := .Values.agent.readreplica -}} +{{- if $agent.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $readreplica.name }}-config + labels: + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} + app.kubernetes.io/component: agent +data: + config.yaml: | + --- + version: {{ $agent.version }} + time_zone: {{ default .Values.defaults.time_zone $agent.time_zone }} + logging: + {{- $logging := dict "Values" $agent.logging "default" .Values.defaults.logging }} + {{- include "vald.logging" $logging | nindent 6 }} + server_config: + {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config }} + {{- include "vald.servers" $servers | nindent 6 }} + observability: + {{- $observability := dict "Values" $agent.observability "default" .Values.defaults.observability }} + {{- include "vald.observability" $observability | nindent 6 }} + ngt: + {{- toYaml $agent.ngt | nindent 6 }} + is_readreplica: true +{{- end }} diff --git a/charts/vald-readreplica/templates/deployment.yaml b/charts/vald-readreplica/templates/deployment.yaml index 5cb2a9a926..880eb8e5ff 100644 --- a/charts/vald-readreplica/templates/deployment.yaml +++ b/charts/vald-readreplica/templates/deployment.yaml @@ -45,7 +45,7 @@ metadata: {{- end }} spec: {{- if not $agent.hpa.enabled }} - replicas: {{ $readreplica.replica }} + replicas: {{ $readreplica.minReplicas }} {{- end }} revisionHistoryLimit: {{ $agent.revisionHistoryLimit }} selector: @@ -110,7 +110,7 @@ spec: {{- toYaml $agent.env | nindent 12 }} {{- end }} volumeMounts: - - name: {{ $agent.name }}-config + - name: {{ $readreplica.name }}-config mountPath: /etc/server/ {{- if not $agent.ngt.enable_in_memory_mode }} {{- if $agent.ngt.index_path }} @@ -136,10 +136,10 @@ spec: {{- end }} terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }} volumes: - - name: {{ $agent.name }}-config + - name: {{ $readreplica.name }}-config configMap: defaultMode: 420 - name: {{ $agent.name }}-config + name: {{ $readreplica.name }}-config - name: {{ $readreplica.volume_name }} persistentVolumeClaim: claimName: {{ $readreplica.name }}-pvc-{{ $id }} diff --git a/charts/vald-readreplica/templates/hpa.yaml b/charts/vald-readreplica/templates/hpa.yaml new file mode 100644 index 0000000000..d0c90c073d --- /dev/null +++ b/charts/vald-readreplica/templates/hpa.yaml @@ -0,0 +1,67 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $agent := .Values.agent -}} +{{- $readreplica := .Values.agent.readreplica -}} +{{- $defaults := .Values.defaults -}} +{{- $release := .Release -}} +{{- $chart := .Chart -}} +{{- $valdname := include "vald.name" . -}} +{{- $valdchart := include "vald.chart" . -}} +{{- $cap := .Capabilities -}} +{{- if and $readreplica.enabled $readreplica.hpa.enabled }} +{{ range $id := until (int $agent.minReplicas) }} +--- +{{- if ($cap.APIVersions.Has "autoscaling/v2") }} +apiVersion: autoscaling/v2 +{{- else if ($cap.APIVersions.Has "autoscaling/v1") }} +apiVersion: autoscaling/v1 +{{- else if ($cap.APIVersions.Has "autoscaling/v2beta2") }} +apiVersion: autoscaling/v2beta2 +{{- else }} +apiVersion: autoscaling/v2beta1 +{{- end }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ $readreplica.name }}-{{ $id }} + labels: + app.kubernetes.io/name: {{ $valdname }} + helm.sh/chart: {{ $valdchart }} + app.kubernetes.io/managed-by: {{ $release.Service }} + app.kubernetes.io/instance: {{ $release.Name }} + app.kubernetes.io/version: {{ $chart.Version }} + app.kubernetes.io/component: {{ $readreplica.component_name }} + {{ $readreplica.label_key }}: "{{ $id }}" +spec: + maxReplicas: {{ $readreplica.maxReplicas }} + minReplicas: {{ $readreplica.minReplicas }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $readreplica.name }}-{{ $id }} +{{- if or ($cap.APIVersions.Has "autoscaling/v2") ($cap.APIVersions.Has "autoscaling/v2beta2") ($cap.APIVersions.Has "autoscaling/v2beta1") }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ $readreplica.hpa.targetCPUUtilizationPercentage }} +{{ else }} + targetCPUUtilizationPercentage: {{ $readreplica.hpa.targetCPUUtilizationPercentage }} +{{- end }} +status: +{{- end }} +{{- end }} diff --git a/charts/vald/README.md b/charts/vald/README.md index 831aba25dc..a9bd03c32e 100644 --- a/charts/vald/README.md +++ b/charts/vald/README.md @@ -109,12 +109,15 @@ Run the following command to install the chart, | agent.podPriority.value | int | `1000000000` | agent pod PriorityClass value | | agent.podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | | agent.progressDeadlineSeconds | int | `600` | progress deadline seconds | -| agent.readreplica | object | `{"component_name":"agent-readreplica","enabled":false,"label_key":"vald-readreplica-id","name":"vald-agent-ngt-readreplica","replica":2,"service":{"annotations":{}},"snapshot_classname":"","volume_name":"vald-agent-ngt-readreplica-pvc"}` | readreplica deployment annotations | +| agent.readreplica | object | `{"component_name":"agent-readreplica","enabled":false,"hpa":{"enabled":false,"targetCPUUtilizationPercentage":80},"label_key":"vald-readreplica-id","maxReplicas":3,"minReplicas":1,"name":"vald-agent-ngt-readreplica","service":{"annotations":{}},"snapshot_classname":"","volume_name":"vald-agent-ngt-readreplica-pvc"}` | readreplica deployment annotations | | agent.readreplica.component_name | string | `"agent-readreplica"` | app.kubernetes.io/component name of agent readreplica | | agent.readreplica.enabled | bool | `false` | [This feature is WORK IN PROGRESS]enable agent readreplica | +| agent.readreplica.hpa.enabled | bool | `false` | HPA enabled | +| agent.readreplica.hpa.targetCPUUtilizationPercentage | int | `80` | HPA CPU utilization percentage | | agent.readreplica.label_key | string | `"vald-readreplica-id"` | label key to identify read replica resources | +| agent.readreplica.maxReplicas | int | `3` | maximum number of replicas. if HPA is disabled, this value will be ignored. | +| agent.readreplica.minReplicas | int | `1` | minimum number of replicas. if HPA is disabled, the replicas will be set to this value | | agent.readreplica.name | string | `"vald-agent-ngt-readreplica"` | name of agent readreplica | -| agent.readreplica.replica | int | `2` | replica number of read replica | | agent.readreplica.service | object | `{"annotations":{}}` | service settings for read replica service resources | | agent.readreplica.service.annotations | object | `{}` | readreplica deployment annotations | | agent.readreplica.snapshot_classname | string | `""` | snapshot class name for snapshotter used for read replica | @@ -663,6 +666,7 @@ Run the following command to install the chart, | gateway.lb.gateway_config.discoverer.agent_client_options | object | `{}` | gRPC client options for agents (overrides defaults.grpc.client) | | gateway.lb.gateway_config.discoverer.client | object | `{}` | gRPC client for discoverer (overrides defaults.grpc.client) | | gateway.lb.gateway_config.discoverer.duration | string | `"200ms"` | | +| gateway.lb.gateway_config.discoverer.read_client | object | `{}` | gRPC client for discoverer (overrides defaults.grpc.client) | | gateway.lb.gateway_config.index_replica | int | `3` | number of index replica | | gateway.lb.gateway_config.multi_operation_concurrency | int | `20` | number of concurrency of multiXXX api's operation | | gateway.lb.gateway_config.node_name | string | `""` | node name | @@ -710,6 +714,95 @@ Run the following command to install the chart, | gateway.lb.version | string | `"v0.0.0"` | version of gateway config | | gateway.lb.volumeMounts | list | `[]` | volume mounts | | gateway.lb.volumes | list | `[]` | volumes | +| gateway.mirror.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | +| gateway.mirror.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | +| gateway.mirror.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | +| gateway.mirror.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity required scheduling terms | +| gateway.mirror.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["vald-mirror-gateway"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | pod anti-affinity preferred scheduling terms | +| gateway.mirror.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | +| gateway.mirror.annotations | object | `{}` | deployment annotations | +| gateway.mirror.clusterRole.enabled | bool | `true` | creates clusterRole resource | +| gateway.mirror.clusterRole.name | string | `"gateway-mirror"` | name of clusterRole | +| gateway.mirror.clusterRoleBinding.enabled | bool | `true` | creates clusterRoleBinding resource | +| gateway.mirror.clusterRoleBinding.name | string | `"gateway-mirror"` | name of clusterRoleBinding | +| gateway.mirror.enabled | bool | `false` | gateway enabled | +| gateway.mirror.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | +| gateway.mirror.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| gateway.mirror.gateway_config.client | object | `{}` | gRPC client (overrides defaults.grpc.client) | +| gateway.mirror.gateway_config.colocation | string | `"dc1"` | colocation name | +| gateway.mirror.gateway_config.discovery_duration | string | `"1s"` | duration to discovery | +| gateway.mirror.gateway_config.gateway_addr | string | `""` | address for lb-gateway | +| gateway.mirror.gateway_config.group | string | `""` | mirror group name | +| gateway.mirror.gateway_config.namespace | string | `"_MY_POD_NAMESPACE_"` | namespace to discovery | +| gateway.mirror.gateway_config.net.dialer.dual_stack_enabled | bool | `false` | TCP dialer dual stack enabled | +| gateway.mirror.gateway_config.net.dialer.keepalive | string | `"10m"` | TCP dialer keep alive | +| gateway.mirror.gateway_config.net.dialer.timeout | string | `"30s"` | TCP dialer timeout | +| gateway.mirror.gateway_config.net.dns.cache_enabled | bool | `true` | TCP DNS cache enabled | +| gateway.mirror.gateway_config.net.dns.cache_expiration | string | `"24h"` | TCP DNS cache expiration | +| gateway.mirror.gateway_config.net.dns.refresh_duration | string | `"5m"` | TCP DNS cache refresh duration | +| gateway.mirror.gateway_config.net.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| gateway.mirror.gateway_config.net.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| gateway.mirror.gateway_config.net.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| gateway.mirror.gateway_config.net.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| gateway.mirror.gateway_config.net.socket_option.tcp_cork | bool | `false` | server listen socket option for tcp_cork functionality | +| gateway.mirror.gateway_config.net.socket_option.tcp_defer_accept | bool | `true` | server listen socket option for tcp_defer_accept functionality | +| gateway.mirror.gateway_config.net.socket_option.tcp_fast_open | bool | `true` | server listen socket option for tcp_fast_open functionality | +| gateway.mirror.gateway_config.net.socket_option.tcp_no_delay | bool | `true` | server listen socket option for tcp_no_delay functionality | +| gateway.mirror.gateway_config.net.socket_option.tcp_quick_ack | bool | `true` | server listen socket option for tcp_quick_ack functionality | +| gateway.mirror.gateway_config.net.tls.ca | string | `"/path/to/ca"` | TLS ca path | +| gateway.mirror.gateway_config.net.tls.cert | string | `"/path/to/cert"` | TLS cert path | +| gateway.mirror.gateway_config.net.tls.enabled | bool | `false` | TLS enabled | +| gateway.mirror.gateway_config.net.tls.insecure_skip_verify | bool | `false` | enable/disable skip SSL certificate verification | +| gateway.mirror.gateway_config.net.tls.key | string | `"/path/to/key"` | TLS key path | +| gateway.mirror.gateway_config.pod_name | string | `"_MY_POD_NAME_"` | self mirror gateway pod name | +| gateway.mirror.gateway_config.register_duration | string | `"1s"` | duration to register mirror-gateway. | +| gateway.mirror.gateway_config.self_mirror_addr | string | `""` | address for self mirror-gateway | +| gateway.mirror.hpa.enabled | bool | `true` | HPA enabled | +| gateway.mirror.hpa.targetCPUUtilizationPercentage | int | `80` | HPA CPU utilization percentage | +| gateway.mirror.image.pullPolicy | string | `"Always"` | image pull policy | +| gateway.mirror.image.repository | string | `"vdaas/vald-mirror-gateway"` | image repository | +| gateway.mirror.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| gateway.mirror.ingress.annotations | object | `{"nginx.ingress.kubernetes.io/grpc-backend":"true"}` | annotations for ingress | +| gateway.mirror.ingress.defaultBackend | object | `{"enabled":true}` | defaultBackend config | +| gateway.mirror.ingress.defaultBackend.enabled | bool | `true` | gateway ingress defaultBackend enabled | +| gateway.mirror.ingress.enabled | bool | `false` | gateway ingress enabled | +| gateway.mirror.ingress.host | string | `"mirror.gateway.vald.vdaas.org"` | ingress hostname | +| gateway.mirror.ingress.pathType | string | `"ImplementationSpecific"` | gateway ingress pathType | +| gateway.mirror.ingress.servicePort | string | `"grpc"` | service port to be exposed by ingress | +| gateway.mirror.initContainers | list | `[{"image":"busybox:stable","name":"wait-for-gateway-lb","sleepDuration":2,"target":"gateway-lb","type":"wait-for"}]` | init containers | +| gateway.mirror.internalTrafficPolicy | string | `""` | internal traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| gateway.mirror.kind | string | `"Deployment"` | deployment kind: Deployment or DaemonSet | +| gateway.mirror.logging | object | `{}` | logging config (overrides defaults.logging) | +| gateway.mirror.maxReplicas | int | `9` | maximum number of replicas. if HPA is disabled, this value will be ignored. | +| gateway.mirror.maxUnavailable | string | `"50%"` | maximum number of unavailable replicas | +| gateway.mirror.minReplicas | int | `3` | minimum number of replicas. if HPA is disabled, the replicas will be set to this value | +| gateway.mirror.name | string | `"vald-mirror-gateway"` | name of gateway deployment | +| gateway.mirror.nodeName | string | `""` | node name | +| gateway.mirror.nodeSelector | object | `{}` | node selector | +| gateway.mirror.observability | object | `{"otlp":{"attribute":{"service_name":"vald-mirror-gateway"}}}` | observability config (overrides defaults.observability) | +| gateway.mirror.podAnnotations | object | `{}` | pod annotations | +| gateway.mirror.podPriority.enabled | bool | `true` | gateway pod PriorityClass enabled | +| gateway.mirror.podPriority.value | int | `1000000` | gateway pod PriorityClass value | +| gateway.mirror.podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | +| gateway.mirror.progressDeadlineSeconds | int | `600` | progress deadline seconds | +| gateway.mirror.resources | object | `{"limits":{"cpu":"2000m","memory":"700Mi"},"requests":{"cpu":"200m","memory":"150Mi"}}` | compute resources | +| gateway.mirror.revisionHistoryLimit | int | `2` | number of old history to retain to allow rollback | +| gateway.mirror.rollingUpdate.maxSurge | string | `"25%"` | max surge of rolling update | +| gateway.mirror.rollingUpdate.maxUnavailable | string | `"25%"` | max unavailable of rolling update | +| gateway.mirror.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for container | +| gateway.mirror.server_config | object | `{"healths":{"liveness":{},"readiness":{},"startup":{}},"metrics":{"pprof":{}},"servers":{"grpc":{},"rest":{}}}` | server config (overrides defaults.server_config) | +| gateway.mirror.service.annotations | object | `{}` | service annotations | +| gateway.mirror.service.labels | object | `{}` | service labels | +| gateway.mirror.serviceAccount.enabled | bool | `true` | creates service account | +| gateway.mirror.serviceAccount.name | string | `"gateway-mirror"` | name of service account | +| gateway.mirror.serviceType | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| gateway.mirror.terminationGracePeriodSeconds | int | `30` | duration in seconds pod needs to terminate gracefully | +| gateway.mirror.time_zone | string | `""` | Time zone | +| gateway.mirror.tolerations | list | `[]` | tolerations | +| gateway.mirror.topologySpreadConstraints | list | `[]` | topology spread constraints of gateway pods | +| gateway.mirror.version | string | `"v0.0.0"` | version of gateway config | +| gateway.mirror.volumeMounts | list | `[]` | volume mounts | +| gateway.mirror.volumes | list | `[]` | volumes | | manager.index.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | | manager.index.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | | manager.index.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | diff --git a/charts/vald/templates/gateway/lb/configmap.yaml b/charts/vald/templates/gateway/lb/configmap.yaml index 7c840cb0c2..f26cd53ed1 100644 --- a/charts/vald/templates/gateway/lb/configmap.yaml +++ b/charts/vald/templates/gateway/lb/configmap.yaml @@ -51,7 +51,7 @@ data: agent_namespace: {{ $gateway.gateway_config.agent_namespace | quote }} node_name: {{ $gateway.gateway_config.node_name | quote }} index_replica: {{ $gateway.gateway_config.index_replica }} - read_replica_replicas: {{ $readreplica.replica }} + read_replica_replicas: {{ $readreplica.minReplicas }} discoverer: duration: {{ $gateway.gateway_config.discoverer.duration }} client: @@ -70,7 +70,7 @@ data: {{- if $readreplica.enabled }} read_replica_client: client: - {{- $discovererClient := $gateway.gateway_config.discoverer.client }} + {{- $discovererClient := $gateway.gateway_config.discoverer.read_client }} {{- $readReplicaPort := $agent.server_config.servers.grpc.port }} {{- $defaultReadReplicaPort := default .Values.defaults.server_config.servers.grpc.port $readReplicaPort }} {{- $readReplicaAddrs := list }} diff --git a/charts/vald/values.schema.json b/charts/vald/values.schema.json index 41cd419a73..551f638a77 100644 --- a/charts/vald/values.schema.json +++ b/charts/vald/values.schema.json @@ -475,18 +475,34 @@ "type": "boolean", "description": "[This feature is WORK IN PROGRESS]enable agent readreplica" }, + "hpa": { + "type": "object", + "properties": { + "enabled": { "type": "boolean", "description": "HPA enabled" }, + "targetCPUUtilizationPercentage": { + "type": "integer", + "description": "HPA CPU utilization percentage" + } + } + }, "label_key": { "type": "string", "description": "label key to identify read replica resources" }, + "maxReplicas": { + "type": "integer", + "description": "maximum number of replicas. if HPA is disabled, this value will be ignored.", + "minimum": 1 + }, + "minReplicas": { + "type": "integer", + "description": "minimum number of replicas. if HPA is disabled, the replicas will be set to this value", + "minimum": 1 + }, "name": { "type": "string", "description": "name of agent readreplica" }, - "replica": { - "type": "integer", - "description": "replica number of read replica" - }, "service": { "type": "object", "description": "service settings for read replica service resources", @@ -8819,7 +8835,321 @@ "wait_for_ready": { "type": "boolean" } } }, - "duration": { "type": "string" } + "duration": { "type": "string" }, + "read_client": { + "type": "object", + "properties": { + "addrs": { + "type": "array", + "description": "gRPC client addresses", + "items": { "type": "string" } + }, + "backoff": { + "type": "object", + "properties": { + "backoff_factor": { + "type": "number", + "description": "gRPC client backoff factor" + }, + "backoff_time_limit": { + "type": "string", + "description": "gRPC client backoff time limit" + }, + "enable_error_log": { + "type": "boolean", + "description": "gRPC client backoff log enabled" + }, + "initial_duration": { + "type": "string", + "description": "gRPC client backoff initial duration" + }, + "jitter_limit": { + "type": "string", + "description": "gRPC client backoff jitter limit" + }, + "maximum_duration": { + "type": "string", + "description": "gRPC client backoff maximum duration" + }, + "retry_count": { + "type": "integer", + "description": "gRPC client backoff retry count" + } + } + }, + "call_option": { "type": "object" }, + "circuit_breaker": { + "type": "object", + "properties": { + "closed_error_rate": { + "type": "number", + "description": "gRPC client circuitbreaker closed error rate" + }, + "closed_refresh_timeout": { + "type": "string", + "description": "gRPC client circuitbreaker closed refresh timeout" + }, + "half_open_error_rate": { + "type": "number", + "description": "gRPC client circuitbreaker half-open error rate" + }, + "min_samples": { + "type": "integer", + "description": "gRPC client circuitbreaker minimum sampling count" + }, + "open_timeout": { + "type": "string", + "description": "gRPC client circuitbreaker open timeout" + } + } + }, + "connection_pool": { + "type": "object", + "properties": { + "enable_dns_resolver": { + "type": "boolean", + "description": "enables gRPC client connection pool dns resolver, when enabled vald uses ip handshake exclude dns discovery which improves network performance" + }, + "enable_rebalance": { + "type": "boolean", + "description": "enables gRPC client connection pool rebalance" + }, + "old_conn_close_duration": { + "type": "string", + "description": "makes delay before gRPC client connection closing during connection pool rebalance" + }, + "rebalance_duration": { + "type": "string", + "description": "gRPC client connection pool rebalance duration" + }, + "size": { + "type": "integer", + "description": "gRPC client connection pool size" + } + } + }, + "dial_option": { + "type": "object", + "properties": { + "backoff_base_delay": { + "type": "string", + "description": "gRPC client dial option base backoff delay" + }, + "backoff_jitter": { + "type": "number", + "description": "gRPC client dial option base backoff delay" + }, + "backoff_max_delay": { + "type": "string", + "description": "gRPC client dial option max backoff delay" + }, + "backoff_multiplier": { + "type": "number", + "description": "gRPC client dial option base backoff delay" + }, + "enable_backoff": { + "type": "boolean", + "description": "gRPC client dial option backoff enabled" + }, + "initial_connection_window_size": { + "type": "integer", + "description": "gRPC client dial option initial connection window size" + }, + "initial_window_size": { + "type": "integer", + "description": "gRPC client dial option initial window size" + }, + "insecure": { + "type": "boolean", + "description": "gRPC client dial option insecure enabled" + }, + "interceptors": { + "type": "array", + "description": "gRPC client interceptors", + "items": { + "type": "string", + "enum": ["TraceInterceptor"] + } + }, + "keepalive": { + "type": "object", + "properties": { + "permit_without_stream": { + "type": "boolean", + "description": "gRPC client keep alive permit without stream" + }, + "time": { + "type": "string", + "description": "gRPC client keep alive time" + }, + "timeout": { + "type": "string", + "description": "gRPC client keep alive timeout" + } + } + }, + "max_msg_size": { + "type": "integer", + "description": "gRPC client dial option max message size" + }, + "min_connection_timeout": { + "type": "string", + "description": "gRPC client dial option minimum connection timeout" + }, + "net": { + "type": "object", + "properties": { + "dialer": { + "type": "object", + "properties": { + "dual_stack_enabled": { + "type": "boolean", + "description": "gRPC client TCP dialer dual stack enabled" + }, + "keepalive": { + "type": "string", + "description": "gRPC client TCP dialer keep alive" + }, + "timeout": { + "type": "string", + "description": "gRPC client TCP dialer timeout" + } + } + }, + "dns": { + "type": "object", + "properties": { + "cache_enabled": { + "type": "boolean", + "description": "gRPC client TCP DNS cache enabled" + }, + "cache_expiration": { + "type": "string", + "description": "gRPC client TCP DNS cache expiration" + }, + "refresh_duration": { + "type": "string", + "description": "gRPC client TCP DNS cache refresh duration" + } + } + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "tls": { + "type": "object", + "properties": { + "ca": { + "type": "string", + "description": "TLS ca path" + }, + "cert": { + "type": "string", + "description": "TLS cert path" + }, + "enabled": { + "type": "boolean", + "description": "TLS enabled" + }, + "insecure_skip_verify": { + "type": "boolean", + "description": "enable/disable skip SSL certificate verification" + }, + "key": { + "type": "string", + "description": "TLS key path" + } + } + } + } + }, + "read_buffer_size": { + "type": "integer", + "description": "gRPC client dial option read buffer size" + }, + "timeout": { + "type": "string", + "description": "gRPC client dial option timeout" + }, + "write_buffer_size": { + "type": "integer", + "description": "gRPC client dial option write buffer size" + } + } + }, + "health_check_duration": { + "type": "string", + "description": "gRPC client health check duration" + }, + "max_recv_msg_size": { "type": "integer" }, + "max_retry_rpc_buffer_size": { "type": "integer" }, + "max_send_msg_size": { "type": "integer" }, + "tls": { + "type": "object", + "properties": { + "ca": { + "type": "string", + "description": "TLS ca path" + }, + "cert": { + "type": "string", + "description": "TLS cert path" + }, + "enabled": { + "type": "boolean", + "description": "TLS enabled" + }, + "insecure_skip_verify": { + "type": "boolean", + "description": "enable/disable skip SSL certificate verification" + }, + "key": { + "type": "string", + "description": "TLS key path" + } + } + }, + "wait_for_ready": { "type": "boolean" } + } + } } }, "index_replica": { diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 312fc09df7..0c3cda4520 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -1125,6 +1125,9 @@ gateway: # @schema {"name": "gateway.lb.gateway_config.discoverer.agent_client_options", "alias": "grpc.client"} # gateway.lb.gateway_config.discoverer.agent_client_options -- gRPC client options for agents (overrides defaults.grpc.client) agent_client_options: {} + # @schema {"name": "gateway.lb.gateway_config.discoverer.read_client", "alias": "grpc.client"} + # gateway.lb.gateway_config.discoverer.read_client -- gRPC client for discoverer (overrides defaults.grpc.client) + read_client: {} # @schema {"name": "gateway.filter", "type": "object"} filter: # @schema {"name": "gateway.filter.enabled", "type": "boolean"} @@ -2003,9 +2006,20 @@ agent: # @schema {"name": "agent.readreplica.snapshot_classname", "type": "string"} # agent.readreplica.snapshot_classname -- snapshot class name for snapshotter used for read replica snapshot_classname: "" - # @schema {"name": "agent.readreplica.replica", "type": "integer"} - # agent.readreplica.replica -- replica number of read replica - replica: 1 + # @schema {"name": "agent.readreplica.minReplicas", "type": "integer", "minimum": 1} + # agent.readreplica.minReplicas -- minimum number of replicas. + # if HPA is disabled, the replicas will be set to this value + minReplicas: 1 + # @schema {"name": "agent.readreplica.maxReplicas", "type": "integer", "minimum": 1} + # agent.readreplica.maxReplicas -- maximum number of replicas. + # if HPA is disabled, this value will be ignored. + maxReplicas: 3 + # @schema {"name": "agent.readreplica.hpa", "alias": "hpa"} + hpa: + # agent.readreplica.hpa.enabled -- HPA enabled + enabled: false + # agent.readreplica.hpa.targetCPUUtilizationPercentage -- HPA CPU utilization percentage + targetCPUUtilizationPercentage: 80 # @schema {"name": "agent.readreplica.service", "type": "object"} # agent.readreplica.service -- service settings for read replica service resources service: diff --git a/internal/config/ngt.go b/internal/config/ngt.go index d277b3b2a6..6a9d8add9d 100644 --- a/internal/config/ngt.go +++ b/internal/config/ngt.go @@ -94,6 +94,9 @@ type NGT struct { // ErrorBufferLimit represents the maximum number of core ngt error buffer pool size limit ErrorBufferLimit uint64 `yaml:"error_buffer_limit" json:"error_buffer_limit,omitempty"` + + // IsReadReplica represents whether the ngt is read replica or not + IsReadReplica bool `yaml:"is_readreplica" json:"is_readreplica"` } // KVSDB represent the ngt vector bidirectional kv store configuration diff --git a/internal/errors/agent.go b/internal/errors/agent.go index 3286a086ef..01d89ce9be 100644 --- a/internal/errors/agent.go +++ b/internal/errors/agent.go @@ -112,4 +112,7 @@ var ( // ErrAgentIndexDirectoryRecreationFailed represents an error that the index directory recreation failed during the process of broken index backup. ErrIndexDirectoryRecreationFailed = New("failed to recreate the index directory") + + // ErrWriteOperationToReadReplica represents an error that when a write operation is made to read replica. + ErrWriteOperationToReadReplica = New("write operation to read replica is not possible") ) diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index 6cdae59b60..004b7db253 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -129,7 +129,6 @@ type ngt struct { basePath string // index base directory for CoW brokenPath string // backup broken index path cowmu sync.Mutex // copy on write move lock - backupGen uint64 // number of backup generation poolSize uint32 // default pool size radius float32 // default radius @@ -140,6 +139,8 @@ type ngt struct { kvsdbConcurrency int // kvsdb concurrency historyLimit int // the maximum generation number of broken index backup + + isReadReplica bool } const ( @@ -1101,6 +1102,11 @@ func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) { span.End() } }() + + if n.isReadReplica { + return errors.ErrWriteOperationToReadReplica + } + ic := n.vq.IVQLen() + n.vq.DVQLen() if ic == 0 { return errors.ErrUncommittedIndexNotFound @@ -1270,6 +1276,11 @@ func (n *ngt) SaveIndex(ctx context.Context) (err error) { } func (n *ngt) saveIndex(ctx context.Context) (err error) { + // Skip it here in case this private function is called directly from someone + if n.isReadReplica { + return errors.ErrWriteOperationToReadReplica + } + nocie := atomic.LoadUint64(&n.nocie) if atomic.LoadUint64(&n.lastNocie) == nocie { return @@ -1656,8 +1667,14 @@ func (n *ngt) GetDimensionSize() int { } func (n *ngt) Close(ctx context.Context) (err error) { + defer n.core.Close() + err = n.kvs.Close() if len(n.path) != 0 { + if n.isReadReplica { + log.Info("skip create and save index operation on close because this is read replica") + return err + } cerr := n.CreateIndex(ctx, n.poolSize) if cerr != nil && !errors.Is(err, errors.ErrUncommittedIndexNotFound) && @@ -1681,8 +1698,7 @@ func (n *ngt) Close(ctx context.Context) (err error) { } } } - n.core.Close() - return + return err } func (n *ngt) BrokenIndexCount() uint64 { diff --git a/pkg/agent/core/ngt/service/ngt_test.go b/pkg/agent/core/ngt/service/ngt_test.go index 2aa7b21367..1029cd67a4 100644 --- a/pkg/agent/core/ngt/service/ngt_test.go +++ b/pkg/agent/core/ngt/service/ngt_test.go @@ -63,6 +63,7 @@ var defaultConfig = config.NGT{ Concurrency: 10, }, BrokenIndexHistoryLimit: 1, + ErrorBufferLimit: 100, } type index struct { @@ -921,6 +922,210 @@ func testFoundInBothIvqAndDvq(t *testing.T) { require.Equal(t, err.Error(), want.Error()) } +func Test_ngt_CreateIndex(t *testing.T) { + t.Parallel() + + type args struct { + cfg *config.NGT + opts []Option + } + type test struct { + name string + args args + want error + } + + setup := func(t *testing.T) string { + tmpDir := t.TempDir() + testIndexDir := testdata.GetTestdataPath(testdata.ValidIndex) + err := file.CopyDir(context.Background(), testIndexDir, tmpDir) + require.NoError(t, err) + + return tmpDir + } + + tests := []test{ + func() test { + tmpDir := setup(t) + return test{ + name: "CreateIndex returns ErrUncommittedIndexNotFound when there is nothing uncommitted", + args: args{ + cfg: &defaultConfig, + opts: []Option{ + WithIndexPath(tmpDir), + WithIsReadReplica(false), + }, + }, + want: errors.ErrUncommittedIndexNotFound, + } + }(), + func() test { + tmpDir := setup(t) + return test{ + name: "CreateIndex returns ErrWriteOperationToReadReplica when try to create index to read replica", + args: args{ + cfg: &defaultConfig, + opts: []Option{ + WithIndexPath(tmpDir), + WithIsReadReplica(true), + }, + }, + want: errors.ErrWriteOperationToReadReplica, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + + ngt, err := New(test.args.cfg, test.args.opts...) + require.NoError(tt, err) + + err = ngt.CreateIndex(context.Background(), test.args.cfg.DefaultPoolSize) + require.Equal(tt, test.want, err) + }) + } +} + +func Test_ngt_SaveIndex(t *testing.T) { + t.Parallel() + + type args struct { + cfg *config.NGT + opts []Option + } + type test struct { + name string + args args + want error + } + + setup := func(t *testing.T) string { + tmpDir := t.TempDir() + testIndexDir := testdata.GetTestdataPath(testdata.ValidIndex) + err := file.CopyDir(context.Background(), testIndexDir, tmpDir) + require.NoError(t, err) + + return tmpDir + } + + tests := []test{ + func() test { + tmpDir := setup(t) + return test{ + name: "CreateIndex successes when there is nothing to save", + args: args{ + cfg: &defaultConfig, + opts: []Option{ + WithIndexPath(tmpDir), + WithIsReadReplica(false), + }, + }, + want: nil, + } + }(), + func() test { + tmpDir := setup(t) + return test{ + name: "SaveIndex returns ErrWriteOperationToReadReplica when try to save index to read replica", + args: args{ + cfg: &defaultConfig, + opts: []Option{ + WithIndexPath(tmpDir), + WithIsReadReplica(true), + }, + }, + want: errors.ErrWriteOperationToReadReplica, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + + ngt, err := New(test.args.cfg, test.args.opts...) + require.NoError(tt, err) + + err = ngt.SaveIndex(context.Background()) + require.Equal(tt, test.want, err) + }) + } +} + +func Test_ngt_Close(t *testing.T) { + t.Parallel() + + type args struct { + cfg *config.NGT + opts []Option + } + type test struct { + name string + args args + want error + } + + setup := func(t *testing.T) string { + tmpDir := t.TempDir() + testIndexDir := testdata.GetTestdataPath(testdata.ValidIndex) + err := file.CopyDir(context.Background(), testIndexDir, tmpDir) + require.NoError(t, err) + + return tmpDir + } + + tests := []test{ + func() test { + tmpDir := setup(t) + return test{ + name: "Close returns ErrUncommittedIndexNotFound when it is not a read replica and try to Create Index because nothing has committed", + args: args{ + cfg: &defaultConfig, + opts: []Option{ + WithIndexPath(tmpDir), + WithIsReadReplica(false), + }, + }, + want: errors.ErrUncommittedIndexNotFound, + } + }(), + func() test { + tmpDir := setup(t) + return test{ + name: "Close successes when it is a read replica because it skips all the Close operations", + args: args{ + cfg: &defaultConfig, + opts: []Option{ + WithIndexPath(tmpDir), + WithIsReadReplica(true), + }, + }, + want: nil, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + + ngt, err := New(test.args.cfg, test.args.opts...) + require.NoError(tt, err) + + err = ngt.Close(context.Background()) + require.Equal(tt, test.want, err) + }) + } +} + func Test_ngt_InsertUpsert(t *testing.T) { if testing.Short() { t.Skip("The execution of this test takes a lot of time, so it is not performed during the short test\ttest: Test_ngt_InsertUpsert") diff --git a/pkg/agent/core/ngt/service/option.go b/pkg/agent/core/ngt/service/option.go index 17bcdaa98a..301c66c621 100644 --- a/pkg/agent/core/ngt/service/option.go +++ b/pkg/agent/core/ngt/service/option.go @@ -299,3 +299,11 @@ func WithCopyOnWrite(enabled bool) Option { return nil } } + +// WithIsReadReplica returns the functional option to set the read replica flag. +func WithIsReadReplica(isReadReplica bool) Option { + return func(n *ngt) error { + n.isReadReplica = isReadReplica + return nil + } +} diff --git a/pkg/agent/core/ngt/usecase/agentd.go b/pkg/agent/core/ngt/usecase/agentd.go index db97ea7f90..ff5a6ee0ab 100644 --- a/pkg/agent/core/ngt/usecase/agentd.go +++ b/pkg/agent/core/ngt/usecase/agentd.go @@ -66,6 +66,7 @@ func New(cfg *config.Data) (r runner.Runner, err error) { service.WithDefaultEpsilon(cfg.NGT.DefaultEpsilon), service.WithProactiveGC(cfg.NGT.EnableProactiveGC), service.WithCopyOnWrite(cfg.NGT.EnableCopyOnWrite), + service.WithIsReadReplica(cfg.NGT.IsReadReplica), ) if err != nil { return nil, err From 52ca81278fde358952fee6bde366aecdf0a5be41 Mon Sep 17 00:00:00 2001 From: Hiroto Funakoshi Date: Mon, 29 Jan 2024 17:10:14 +0900 Subject: [PATCH 07/10] Change to dynamically switch CI container image tag (#2310) * feat: use ci image tag when ci container image is created Signed-off-by: hlts2 * fix: use container image tag Signed-off-by: hlts2 --------- Signed-off-by: hlts2 --- .github/workflows/build-binaries.yml | 6 +++++- .github/workflows/build-protobuf.yml | 6 +++++- .github/workflows/codeql-analysis.yml | 6 +++++- .github/workflows/coverage.yml | 6 +++++- .github/workflows/e2e-code-bench-agent.yaml | 9 +++++++-- .github/workflows/fossa.yml | 7 +++++-- .github/workflows/helm-lint.yml | 12 ++++++++++-- .github/workflows/helm.yml | 6 +++++- .github/workflows/reviewdog-k8s.yml | 12 +++++++++--- .github/workflows/reviewdog.yml | 6 +++++- 10 files changed, 61 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 4a98c9e045..594b196beb 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -26,10 +26,14 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + build-linux: runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index 4a3cc0df0d..c77aac966d 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -35,10 +35,14 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + build: runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3c02a4aaf5..71e4b58133 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,11 +43,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + codeql-build: name: CodeQL runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f1e7a8c3e0..65956dcc74 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,10 +37,14 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + coverage: runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml index 81b572b14c..659ef33ed3 100644 --- a/.github/workflows/e2e-code-bench-agent.yaml +++ b/.github/workflows/e2e-code-bench-agent.yaml @@ -53,11 +53,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + grpc-sequential: name: grpc-sequential runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 @@ -87,8 +91,9 @@ jobs: grpc-stream: name: grpc-stream runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index 8bb9190eab..689a3ec9b9 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -30,12 +30,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + fossa-scan: name: "trigger FOSSA scan" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest + needs: [dump-contexts-to-log, detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 49aef39f15..87cac5be5d 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -26,11 +26,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + lint-vald-chart: name: lint for vald chart runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - name: Check out code. uses: actions/checkout@v4 @@ -54,8 +58,9 @@ jobs: lint-vald-helm-operator-chart: name: lint for vald-helm-operator chart runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - name: Check out code. uses: actions/checkout@v4 @@ -79,6 +84,9 @@ jobs: lint-values-schema: name: lint for values schema runs-on: ubuntu-latest + needs: [detect-ci-container] + container: + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - name: Check out code. uses: actions/checkout@v4 diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 7e26021309..9e8421a598 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -29,11 +29,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + update-helm-chart: name: Update Helm chart runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/reviewdog-k8s.yml b/.github/workflows/reviewdog-k8s.yml index d5e746c148..f7310207a9 100644 --- a/.github/workflows/reviewdog-k8s.yml +++ b/.github/workflows/reviewdog-k8s.yml @@ -28,11 +28,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + kubeval-conflint: name: runner / kubeval-conflint runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: mumoshu/conflint:latest + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 @@ -53,8 +57,9 @@ jobs: kubelinter-k8s-manifests: name: runner / kubelinter runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 @@ -74,8 +79,9 @@ jobs: kubelinter-vald-chart: name: runner / kubelinter for vald chart runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index 07d627fbd0..a8238a16b8 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -27,11 +27,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context + detect-ci-container: + uses: ./.github/workflows/_detect-ci-container.yml + golangci-lint: name: runner / golangci-lint runs-on: ubuntu-latest + needs: [detect-ci-container] container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly + image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG_NAME }} steps: - uses: actions/checkout@v4 From 6c41eacbaee97d8864d74513b6bde718229a316e Mon Sep 17 00:00:00 2001 From: Kiichiro YUKAWA Date: Tue, 30 Jan 2024 14:00:47 +0900 Subject: [PATCH 08/10] Add search optimization document (#2306) * :pencil: Add search optimization document Signed-off-by: vankichi * :pencil: Fix Signed-off-by: vankichi * :pencil: Add links Signed-off-by: vankichi * :pencil: Fix tag Signed-off-by: vankichi --------- Signed-off-by: vankichi --- docs/performance/tuning-search-performance.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/performance/tuning-search-performance.md diff --git a/docs/performance/tuning-search-performance.md b/docs/performance/tuning-search-performance.md new file mode 100644 index 0000000000..b0cb6586df --- /dev/null +++ b/docs/performance/tuning-search-performance.md @@ -0,0 +1,64 @@ +# Tuning Search Performance + +ANN is fast, but sometimes it can be improved more by tuning parameters. +This page shows the essence for improve ANN search by the Vald cluster. + +## Tuning Guideline + +When the search results do NOT satisfy the expected result, it can be improved by tuning parameters. + +First of all, we recommend tuning by following the steps below without doing it blindly. + +```mermaid +flowchart TD + A[Perform Linear Search API] + B{Is satisfies?} + C[Tuning Parameters to improve precision] + D[Tuning Embedding Models] + E[Perform Search API] + F[Tuning Parameters to improve latency] + A-->B + B-- Yes -->C + B-- No -->D + C--> E + E--> C + E--> F + F--> E +``` + +The best practice is: + +1. Measure [Linear Search](../../docs/api/search.md#linearsearch-rpc) performance and use it as a baseline for Search API +1. Repeat tuning to improve precision and measure Search API until the conditions are met +1. Repeat tuning to improve latency and measure Search API until the conditions are met + +
+When the results are not good by Linear Search API, it may need to rethink the embedding model for vectorization. +
+ +## Tuning parameters + +There are two viewpoints, client-side and cluster-side, for improving search performance. + +
+There is a trade-off between search speed and accuracy, so tuning accuracy at first is recommended. +
+ +### Client side + +On the client side, parameters of [`Search.Config`](../../docs/api/search.md#input) will affect the search result. + +| | description | how does it affect? | memo | +| :---------- | :---------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------ | +| radius | the search radius for NGT
[ref: search_radius](https://github.com/yahoojapan/NGT/tree/main/bin/ngt#search) | Define the search range when NGT searches the nearest neighbors | recommended value: `-1` | +| epsilon | the search coefficient for NGT
[ref: search_range_coefficient](https://github.com/yahoojapan/NGT/tree/main/bin/ngt#search) | Expansion factor of the NGT search range.
Search operation time increases when the epsilon is big. | recommended value range: `0.01 ~ 0.3`
default value: `0.1` | +| timeout(ns) | max time duration until receiving search results. | An error will be returned if the set `num` search results cannot be obtained within the set time.
By setting `min_num`, the search results will be returned if more than `min_num` can be searched within the time. | default value: `3,000,000,000ns` | + +### Cluster-side + +On the cluster side, these parameters can be set by `values.yaml`, affect the search result. + +| | description | how does it affect? | Memo | +| :--------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------ | +| agent.ngt.creation_edge_size | Number of nodes connected to one node
[ref: no_of_edges](https://github.com/yahoojapan/NGT/tree/main/bin/ngt#create) | It helps reduce unreachable edges.
The larger it is, the denser the graph structure will be, but the memory usage, search speed, and index construction time will increase accordingly. | default value: `20` | +| agent.ngt.search_edge_size | Number of nodes to search from the origin node when searching
[ref: no_of_edges_at_search_at](https://github.com/yahoojapan/NGT/tree/main/bin/ngt#create) | The number of nodes to search will increase.
Accuracy will be higher, but speed will be lower.
Adjust if adjusting the radius and epsilon does not improve the situation. | default value: `10` | From b3958f4cc3b6a5f9992547b00078bfc3abdf631c Mon Sep 17 00:00:00 2001 From: Yusuke Kadowaki Date: Tue, 30 Jan 2024 16:32:50 +0900 Subject: [PATCH 09/10] Refactor postAttachCommand (#2312) * Refactor postAttachCommand * Refactor comment --- .devcontainer/devcontainer.json | 7 ++---- .devcontainer/postAttachCommand.sh | 35 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 .devcontainer/postAttachCommand.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f53cc21711..997606833e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -19,11 +19,8 @@ // To access grafana "appPort": "3000:3000", - // Persist zsh history settings below - // overwrite HISTFILE to store history file in the named volume defined below - // also setting INC_APPEND_HISTORY to immediately flush command history to the file to keep history when rebuilding the devcontainer - "postStartCommand": "echo 'export HISTFILE=/commandhistory/.zsh_history' >> /root/.zshrc && echo 'setopt INC_APPEND_HISTORY' >> /root/.zshrc", // define named volume to store zsh history file "mounts": ["source=zshhistory-named-volume,target=/commandhistory,type=volume"], - "postAttachCommand": "mkdir -p /etc/server && ln -s $(pwd)/cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml" + + "postAttachCommand": ["/bin/bash", ".devcontainer/postAttachCommand.sh"] } diff --git a/.devcontainer/postAttachCommand.sh b/.devcontainer/postAttachCommand.sh new file mode 100644 index 0000000000..b8fee321a4 --- /dev/null +++ b/.devcontainer/postAttachCommand.sh @@ -0,0 +1,35 @@ +#!/bin/bash -eu +# +# This script is executed as postAttachCommand in devcontainer.json +# This script does... +# - create symbolic link of config.yaml for easier development +# - add command history setting to .zshrc to persist history +# + +echo "creating symbolic link of config ZSHRC..." + +LINK_TARGET="$(pwd)/cmd/agent/core/ngt/sample.yaml" +LINK_SRC="/etc/server/config.yaml" + +mkdir -p /etc/server + +if [ ! -e "$LINK_SRC" ]; then + ln -s "$LINK_TARGET" "$LINK_SRC" + echo "created symbolic link: $LINK_SRC -> $LINK_TARGET" +else + echo "$LINK_SRC already exists" +fi + + +echo "adding history setting to .zshrc..." + +LINE1="export HISTFILE=/commandhistory/.zsh_history" +LINE2="setopt INC_APPEND_HISTORY" + +ZSHRC="/root/.zshrc" + +# write only if those lines don't exist +grep -qxF "$LINE1" "$ZSHRC" || echo "$LINE1" >> "$ZSHRC" +grep -qxF "$LINE2" "$ZSHRC" || echo "$LINE2" >> "$ZSHRC" + +echo "added history setting to .zshrc" From aae170755da814f7ffd7a744090a37c64ca075cf Mon Sep 17 00:00:00 2001 From: Yusuke Kadowaki Date: Wed, 31 Jan 2024 13:09:51 +0900 Subject: [PATCH 10/10] Change lincense/gen/main.go to skip shebang (#2313) * Skip shebang * Apply format --- .devcontainer/postAttachCommand.sh | 20 +++++++++++++++++++- .github/workflows/issue-metrics.yaml | 15 +++++++++++++++ hack/license/gen/main.go | 3 ++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.devcontainer/postAttachCommand.sh b/.devcontainer/postAttachCommand.sh index b8fee321a4..2198678504 100644 --- a/.devcontainer/postAttachCommand.sh +++ b/.devcontainer/postAttachCommand.sh @@ -1,4 +1,22 @@ #!/bin/bash -eu + +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +: + # # This script is executed as postAttachCommand in devcontainer.json # This script does... @@ -6,7 +24,7 @@ # - add command history setting to .zshrc to persist history # -echo "creating symbolic link of config ZSHRC..." +echo "creating symbolic link of config..." LINK_TARGET="$(pwd)/cmd/agent/core/ngt/sample.yaml" LINK_SRC="/etc/server/config.yaml" diff --git a/.github/workflows/issue-metrics.yaml b/.github/workflows/issue-metrics.yaml index d79a13bf59..6c40415a2f 100644 --- a/.github/workflows/issue-metrics.yaml +++ b/.github/workflows/issue-metrics.yaml @@ -1,3 +1,18 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# name: Monthly review time metrics on: workflow_dispatch: diff --git a/hack/license/gen/main.go b/hack/license/gen/main.go index 824c9ee7e1..e348675753 100644 --- a/hack/license/gen/main.go +++ b/hack/license/gen/main.go @@ -307,7 +307,8 @@ func readAndRewrite(path string) error { for sc.Scan() { line := sc.Text() if filepath.Ext(path) == ".go" && strings.HasPrefix(line, "//go:") || - filepath.Ext(path) == ".py" && strings.HasPrefix(line, "# -*-") { + filepath.Ext(path) == ".py" && strings.HasPrefix(line, "# -*-") || + filepath.Ext(path) == ".sh" && strings.HasPrefix(line, "#!") { bf = true _, err = buf.WriteString(line) if err != nil {