diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml index 2c0caa19b07..c37d76495fa 100644 --- a/.github/actions/detect-docker-image-tags/action.yaml +++ b/.github/actions/detect-docker-image-tags/action.yaml @@ -25,6 +25,8 @@ inputs: description: "Image names" required: false default: "vdaas/vald-agent-ngt \ + vdaas/vald-agent-qbg \ + vdaas/vald-agent-sidecar \ vdaas/vald-discoverer-k8s \ vdaas/vald-lb-gateway \ vdaas/vald-manager-index" @@ -45,6 +47,7 @@ runs: run: | declare -A m=( ["vdaas/vald-agent-ngt"]="agent.image.tag" + ["vdaas/vald-agent-qbg"]="agent.image.tag" ["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag" ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag" ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag" diff --git a/.github/actions/wait-for-docker-image/action.yaml b/.github/actions/wait-for-docker-image/action.yaml index 0dac63c4904..8ce474cd1c8 100644 --- a/.github/actions/wait-for-docker-image/action.yaml +++ b/.github/actions/wait-for-docker-image/action.yaml @@ -21,6 +21,8 @@ inputs: description: "image names" required: false default: "vdaas/vald-agent-ngt \ + vdaas/vald-agent-qbg \ + vdaas/vald-agent-sidecar \ vdaas/vald-discoverer-k8s \ vdaas/vald-lb-gateway \ vdaas/vald-manager-index" diff --git a/.github/workflows/dockers-agent-qbg-image.yml b/.github/workflows/dockers-agent-qbg-image.yml new file mode 100644 index 00000000000..e3ad805b021 --- /dev/null +++ b/.github/workflows/dockers-agent-qbg-image.yml @@ -0,0 +1,153 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: agent-qbg" +on: + push: + branches: + - main + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-agent-qbg-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/qbg/**" + - "cmd/agent/core/qbg/**" + - "dockers/agent/core/qbg/Dockerfile" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-agent-qbg-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/qbg/**" + - "cmd/agent/core/qbg/**" + - "dockers/agent/core/qbg/Dockerfile" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-agent-qbg-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/qbg/**" + - "cmd/agent/core/qbg/**" + - "dockers/agent/core/qbg/Dockerfile" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + build: + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) }} + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: agent-qbg + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: agent-qbg image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index 44414247c40..d51d960f807 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -27,9 +27,77 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/dump-context agent-ngt: +<<<<<<< HEAD uses: ./.github/workflows/_docker-image-scan.yaml with: target: agent-ngt +======= + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/agent-ngt + imagename=`make docker/name/agent-ngt` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + agent-qbg: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/agent-qbg + imagename=`make docker/name/agent-qbg` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" +>>>>>>> feature/agent/new-agents-helm-template-base agent-sidecar: uses: ./.github/workflows/_docker-image-scan.yaml with: diff --git a/.gitignore b/.gitignore index fd593b1c6c0..8b35c899ea2 100755 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ hack/go.mod.default3 # for mac .DS_Store +.nvimlog # for nvim .nvimlog diff --git a/Makefile b/Makefile index 7319d6147e3..b5246cd11ca 100644 --- a/Makefile +++ b/Makefile @@ -39,11 +39,15 @@ MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>" VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION) +<<<<<<< HEAD NGT_VERSION := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VERSION) NGT_REPO = github.com/yahoojapan/NGT GOPRIVATE = $(GOPKG),$(GOPKG)/apis,$(GOPKG)-client-go GOPROXY = "https://proxy.golang.org,direct" +======= +GOPROXY=direct +>>>>>>> feature/agent/new-agents-helm-template-base GOPATH := $(eval GOPATH := $(shell go env GOPATH))$(GOPATH) GO_VERSION := $(eval GO_VERSION := $(shell cat versions/GO_VERSION))$(GO_VERSION) GOARCH := $(eval GOARCH := $(shell go env GOARCH))$(GOARCH) @@ -55,8 +59,10 @@ TEST_NOT_IMPL_PLACEHOLDER = NOT IMPLEMENTED BELOW TEMP_DIR := $(eval TEMP_DIR := $(shell mktemp -d))$(TEMP_DIR) -OPERATOR_SDK_VERSION := $(eval OPERATOR_SDK_VERSION := $(shell cat versions/OPERATOR_SDK_VERSION))$(OPERATOR_SDK_VERSION) +NGT_REPO = github.com/yahoojapan/NGT +BUF_VERSION := $(eval BUF_VERSION := $(shell cat versions/BUF_VERSION))$(BUF_VERSION) +FAISS_VERSION := $(eval FAISS_VERSION := $(shell cat versions/FAISS_VERSION))$(FAISS_VERSION) GOLANGCILINT_VERSION := $(eval GOLANGCILINT_VERSION := $(shell cat versions/GOLANGCILINT_VERSION))$(GOLANGCILINT_VERSION) HELM_DOCS_VERSION := $(eval HELM_DOCS_VERSION := $(shell cat versions/HELM_DOCS_VERSION))$(HELM_DOCS_VERSION) HELM_VERSION := $(eval HELM_VERSION := $(shell cat versions/HELM_VERSION))$(HELM_VERSION) @@ -64,6 +70,8 @@ JAEGER_OPERATOR_VERSION := $(eval JAEGER_OPERATOR_VERSION := $(shell cat versi KIND_VERSION := $(eval KIND_VERSION := $(shell cat versions/KIND_VERSION))$(KIND_VERSION) KUBECTL_VERSION := $(eval KUBECTL_VERSION := $(shell cat versions/KUBECTL_VERSION))$(KUBECTL_VERSION) KUBELINTER_VERSION := $(eval KUBELINTER_VERSION := $(shell cat versions/KUBELINTER_VERSION))$(KUBELINTER_VERSION) +NGT_VERSION := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VERSION) +OPERATOR_SDK_VERSION := $(eval OPERATOR_SDK_VERSION := $(shell cat versions/OPERATOR_SDK_VERSION))$(OPERATOR_SDK_VERSION) OTEL_OPERATOR_VERSION := $(eval OTEL_OPERATOR_VERSION := $(shell cat versions/OTEL_OPERATOR_VERSION))$(OTEL_OPERATOR_VERSION) PROMETHEUS_STACK_VERSION := $(eval PROMETHEUS_STACK_VERSION := $(shell cat versions/PROMETHEUS_STACK_VERSION))$(PROMETHEUS_STACK_VERSION) PROTOBUF_VERSION := $(eval PROTOBUF_VERSION := $(shell cat versions/PROTOBUF_VERSION))$(PROTOBUF_VERSION) @@ -71,7 +79,6 @@ REVIEWDOG_VERSION := $(eval REVIEWDOG_VERSION := $(shell cat versions/RE TELEPRESENCE_VERSION := $(eval TELEPRESENCE_VERSION := $(shell cat versions/TELEPRESENCE_VERSION))$(TELEPRESENCE_VERSION) VALDCLI_VERSION := $(eval VALDCLI_VERSION := $(shell cat versions/VALDCLI_VERSION))$(VALDCLI_VERSION) YQ_VERSION := $(eval YQ_VERSION := $(shell cat versions/YQ_VERSION))$(YQ_VERSION) -BUF_VERSION := $(eval BUF_VERSION := $(shell cat versions/BUF_VERSION))$(BUF_VERSION) OTEL_OPERATOR_RELEASE_NAME ?= opentelemetry-operator PROMETHEUS_RELEASE_NAME ?= prometheus @@ -188,29 +195,31 @@ PROTO_PATHS = \ # - internal/test/comparator # - internal/test/mock GO_SOURCES = $(eval GO_SOURCES := $(shell find \ - ./cmd \ - ./hack \ - ./internal \ - ./pkg \ - -not -path './cmd/cli/*' \ - -not -path './internal/core/algorithm/ngt/*' \ - -not -path './internal/compress/gob/*' \ - -not -path './internal/compress/gzip/*' \ - -not -path './internal/compress/lz4/*' \ - -not -path './internal/compress/zstd/*' \ - -not -path './internal/db/storage/blob/s3/sdk/s3/*' \ - -not -path './internal/db/rdb/mysql/dbr/*' \ - -not -path './internal/test/comparator/*' \ - -not -path './internal/test/mock/*' \ - -not -path './hack/benchmark/internal/client/ngtd/*' \ - -not -path './hack/benchmark/internal/starter/agent/*' \ - -not -path './hack/benchmark/internal/starter/external/*' \ - -not -path './hack/benchmark/internal/starter/gateway/*' \ - -not -path './hack/gorules/*' \ - -not -path './hack/license/*' \ - -not -path './hack/swagger/*' \ - -not -path './hack/tools/*' \ - -not -path './tests/*' \ + $(ROOTDIR)/cmd \ + $(ROOTDIR)/hack \ + $(ROOTDIR)/internal \ + $(ROOTDIR)/pkg \ + -not -path '$(ROOTDIR)/cmd/cli/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/ngt/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/qbg/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/faiss/*' \ + -not -path '$(ROOTDIR)/internal/compress/gob/*' \ + -not -path '$(ROOTDIR)/internal/compress/gzip/*' \ + -not -path '$(ROOTDIR)/internal/compress/lz4/*' \ + -not -path '$(ROOTDIR)/internal/compress/zstd/*' \ + -not -path '$(ROOTDIR)/internal/db/storage/blob/s3/sdk/s3/*' \ + -not -path '$(ROOTDIR)/internal/db/rdb/mysql/dbr/*' \ + -not -path '$(ROOTDIR)/internal/test/comparator/*' \ + -not -path '$(ROOTDIR)/internal/test/mock/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/client/ngtd/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/agent/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/external/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/gateway/*' \ + -not -path '$(ROOTDIR)/hack/gorules/*' \ + -not -path '$(ROOTDIR)/hack/license/*' \ + -not -path '$(ROOTDIR)/hack/swagger/*' \ + -not -path '$(ROOTDIR)/hack/tools/*' \ + -not -path '$(ROOTDIR)/tests/*' \ -type f \ -name '*.go' \ -not -regex '.*options?\.go' \ @@ -218,29 +227,31 @@ GO_SOURCES = $(eval GO_SOURCES := $(shell find \ -not -name '*_mock.go' \ -not -name 'doc.go'))$(GO_SOURCES) GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \ - ./cmd \ - ./hack \ - ./internal \ - ./pkg \ - -not -path './cmd/cli/*' \ - -not -path './internal/core/algorithm/ngt/*' \ - -not -path './internal/compress/gob/*' \ - -not -path './internal/compress/gzip/*' \ - -not -path './internal/compress/lz4/*' \ - -not -path './internal/compress/zstd/*' \ - -not -path './internal/db/storage/blob/s3/sdk/s3/*' \ - -not -path './internal/db/rdb/mysql/dbr/*' \ - -not -path './internal/test/comparator/*' \ - -not -path './internal/test/mock/*' \ - -not -path './hack/benchmark/internal/client/ngtd/*' \ - -not -path './hack/benchmark/internal/starter/agent/*' \ - -not -path './hack/benchmark/internal/starter/external/*' \ - -not -path './hack/benchmark/internal/starter/gateway/*' \ - -not -path './hack/gorules/*' \ - -not -path './hack/license/*' \ - -not -path './hack/swagger/*' \ - -not -path './hack/tools/*' \ - -not -path './tests/*' \ + $(ROOTDIR)/cmd \ + $(ROOTDIR)/hack \ + $(ROOTDIR)/internal \ + $(ROOTDIR)/pkg \ + -not -path '$(ROOTDIR)/cmd/cli/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/ngt/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/qbg/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/faiss/*' \ + -not -path '$(ROOTDIR)/internal/compress/gob/*' \ + -not -path '$(ROOTDIR)/internal/compress/gzip/*' \ + -not -path '$(ROOTDIR)/internal/compress/lz4/*' \ + -not -path '$(ROOTDIR)/internal/compress/zstd/*' \ + -not -path '$(ROOTDIR)/internal/db/storage/blob/s3/sdk/s3/*' \ + -not -path '$(ROOTDIR)/internal/db/rdb/mysql/dbr/*' \ + -not -path '$(ROOTDIR)/internal/test/comparator/*' \ + -not -path '$(ROOTDIR)/internal/test/mock/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/client/ngtd/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/agent/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/external/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/gateway/*' \ + -not -path '$(ROOTDIR)/hack/gorules/*' \ + -not -path '$(ROOTDIR)/hack/license/*' \ + -not -path '$(ROOTDIR)/hack/swagger/*' \ + -not -path '$(ROOTDIR)/hack/tools/*' \ + -not -path '$(ROOTDIR)/tests/*' \ -type f \ -regex '.*options?\.go' \ -not -name '*_test.go' \ @@ -248,7 +259,7 @@ GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \ -not -name 'doc.go'))$(GO_OPTION_SOURCES) GO_SOURCES_INTERNAL = $(eval GO_SOURCES_INTERNAL := $(shell find \ - ./internal \ + $(ROOTDIR)/internal \ -type f \ -name '*.go' \ -not -name '*_test.go' \ @@ -334,39 +345,44 @@ all: clean deps .PHONY: clean ## clean -clean: - rm -rf vendor - go clean -cache -modcache -testcache -i -r - mv ./apis/grpc/v1/vald/vald.go $(TEMP_DIR)/vald.go - mv ./apis/grpc/v1/agent/core/agent.go $(TEMP_DIR)/agent.go - mv ./apis/grpc/v1/payload/interface.go $(TEMP_DIR)/interface.go +clean: \ + clean-generated \ + proto/all \ + deps \ + format + +.PHONY: clean-generated +## clean generated files +clean-generated: + mv $(ROOTDIR)/apis/grpc/v1/vald/vald.go $(TEMP_DIR)/vald.go + mv $(ROOTDIR)/apis/grpc/v1/agent/core/agent.go $(TEMP_DIR)/agent.go + mv $(ROOTDIR)/apis/grpc/v1/payload/interface.go $(TEMP_DIR)/interface.go rm -rf \ - /go/pkg \ - ./*.log \ - ./*.svg \ - ./apis/docs \ - ./apis/swagger \ - ./apis/grpc \ - ./bench \ - ./pprof \ - ./libs \ - $(GOCACHE) \ - ./go.sum \ - ./go.mod - mkdir -p ./apis/grpc/v1/vald - mv $(TEMP_DIR)/vald.go ./apis/grpc/v1/vald/vald.go - mkdir -p ./apis/grpc/v1/agent/core - mv $(TEMP_DIR)/agent.go ./apis/grpc/v1/agent/core/agent.go - mkdir -p ./apis/grpc/v1/payload - mv $(TEMP_DIR)/interface.go ./apis/grpc/v1/payload/interface.go - cp ./hack/go.mod.default ./go.mod + $(ROOTDIR)/*.log \ + $(ROOTDIR)/*.svg \ + $(ROOTDIR)/apis/docs \ + $(ROOTDIR)/apis/swagger \ + $(ROOTDIR)/apis/grpc \ + $(ROOTDIR)/bench \ + $(ROOTDIR)/pprof \ + $(ROOTDIR)/libs + mkdir -p $(ROOTDIR)/apis/grpc/v1/vald + mv $(TEMP_DIR)/vald.go $(ROOTDIR)/apis/grpc/v1/vald/vald.go + mkdir -p $(ROOTDIR)/apis/grpc/v1/agent/core + mv $(TEMP_DIR)/agent.go $(ROOTDIR)/apis/grpc/v1/agent/core/agent.go + mkdir -p $(ROOTDIR)/apis/grpc/v1/payload + mv $(TEMP_DIR)/interface.go $(ROOTDIR)/apis/grpc/v1/payload/interface.go .PHONY: license ## add license to files license: GOPRIVATE=$(GOPRIVATE) \ MAINTAINER=$(MAINTAINER) \ +<<<<<<< HEAD go run -mod=readonly hack/license/gen/main.go $(ROOTDIR) +======= + go run -mod=readonly hack/license/gen/main.go $(ROOTDIR)/ +>>>>>>> feature/agent/new-agents-helm-template-base .PHONY: init ## initialize development environment @@ -388,13 +404,17 @@ tools/install: \ .PHONY: update ## update deps, license, and run golines, gofumpt, goimports update: \ - clean \ + clean-generated \ update/libs \ proto/all \ deps \ +<<<<<<< HEAD update/template \ format \ go/deps +======= + format +>>>>>>> feature/agent/new-agents-helm-template-base .PHONY: format ## format go codes @@ -413,10 +433,10 @@ format/go: \ gofumpt/install \ strictgoimports/install \ goimports/install - find ./ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOBIN)/golines -w -m $(GOLINES_MAX_WIDTH) - find ./ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOBIN)/gofumpt -w - find ./ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOBIN)/strictgoimports -w - find ./ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs $(GOBIN)/goimports -w + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOBIN)/golines -w -m $(GOLINES_MAX_WIDTH) + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOBIN)/gofumpt -w + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOBIN)/strictgoimports -w + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs $(GOBIN)/goimports -w .PHONY: format/go/test ## run golines, gofumpt, goimports for go test files @@ -475,7 +495,8 @@ deps/install: \ strictgoimports/install \ goimports/install \ prettier/install \ - go/deps + go/deps \ + go/example/deps .PHONY: version ## print vald version @@ -536,6 +557,18 @@ ngt/install: /usr/local/include/NGT/Capi.h rm -rf $(TEMP_DIR)/NGT-$(NGT_VERSION) ldconfig +.PHONY: faiss/install +## install Faiss +faiss/install: /usr/local/lib/libfaiss.so +/usr/local/lib/libfaiss.so: + curl -LO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz + tar zxf v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/ + cd $(TEMP_DIR)/faiss-$(FAISS_VERSION) && \ + cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON -B build . && \ + make -C build -j faiss && \ + make -C build install + ldconfig + .PHONY: lint ## run lints lint: vet @@ -549,7 +582,7 @@ vet: .PHONY: docs/lint ## run lint for document docs/lint: - textlint docs/**/*.md + textlint $(ROOTDIR)/docs/**/*.md .PHONY: changelog/update ## update changelog @@ -558,13 +591,13 @@ changelog/update: echo "" >> $(TEMP_DIR)/CHANGELOG.md $(MAKE) -s changelog/next/print >> $(TEMP_DIR)/CHANGELOG.md echo "" >> $(TEMP_DIR)/CHANGELOG.md - tail -n +2 CHANGELOG.md >> $(TEMP_DIR)/CHANGELOG.md - mv -f $(TEMP_DIR)/CHANGELOG.md CHANGELOG.md + tail -n +2 $(ROOTDIR)/CHANGELOG.md >> $(TEMP_DIR)/CHANGELOG.md + mv -f $(TEMP_DIR)/CHANGELOG.md $(ROOTDIR)/CHANGELOG.md .PHONY: changelog/next/print ## print next changelog entry changelog/next/print: - @cat hack/CHANGELOG.template.md | \ + @cat $(ROOTDIR)/hack/CHANGELOG.template.md | \ sed -e 's/{{ version }}/$(VERSION)/g' @echo "$$BODY" diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index 0911abaf4fb..1f759055520 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -49,7 +49,43 @@ cmd/agent/core/ngt/ngt: \ -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ -X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \ - -X '$(GOPKG)/internal/info.NGTVersion=$(NGT_VERSION)' \ + -X '$(GOPKG)/internal/info.AlgorithmInfo=NGT-$(NGT_VERSION)' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ + -buildid=" \ + -mod=readonly \ + -modcacherw \ + -a \ + -tags "cgo osusergo netgo static_build" \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + $@ -version + +cmd/agent/core/faiss/faiss: \ + faiss/install \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find $(ROOTDIR)/cmd/agent/core/faiss -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find $(ROOTDIR)/pkg/agent/core/faiss $(ROOTDIR)/pkg/agent/core/ngt/service/kvs $(ROOTDIR)/pkg/agent/core/ngt/service/vqueue $(ROOTDIR)/pkg/agent/internal -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + CFLAGS="$(CFLAGS)" \ + CXXFLAGS="$(CXXFLAGS)" \ + CGO_ENABLED=1 \ + CGO_CXXFLAGS="-g -Ofast -march=native -DDEBUG=0" \ + CGO_FFLAGS="-g -Ofast -march=native -DDEBUG=0" \ + CGO_LDFLAGS="-g -Ofast -march=native -DDEBUG=0" \ + GO111MODULE=on \ + GOPRIVATE=$(GOPRIVATE) \ + go build \ + --ldflags "-w -linkmode 'external' \ + -extldflags '-fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now $(EXTLDFLAGS)' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.FaissVersion=$(FAISS_VERSION)' \ -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ -buildid=" \ -mod=readonly \ diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index d4e73c2b631..6f3215b2fc2 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -18,6 +18,7 @@ ## update vald libraries including tools update/libs: \ update/chaos-mesh \ + update/faiss \ update/go \ update/golangci-lint \ update/helm \ @@ -148,6 +149,11 @@ update/kube-linter: update/ngt: curl --silent https://api.github.com/repos/yahoojapan/NGT/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/NGT_VERSION +.PHONY: update/faiss +## update facebookresearch/faiss version +update/faiss: + curl --silent https://api.github.com/repos/facebookresearch/faiss/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/FAISS_VERSION + .PHONY: update/reviewdog ## update reviewdog version update/reviewdog: diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 2749e4f0f86..1cb38e022f4 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -83,6 +83,18 @@ docker/build/agent-ngt: IMAGE=$(AGENT_IMAGE) \ docker/build/image +.PHONY: docker/build/agent-faiss +## build agent-faiss image +docker/build/agent-faiss: + $(DOCKER) build \ + $(DOCKER_OPTS) \ + -f dockers/agent/core/faiss/Dockerfile \ + -t $(ORG)/vald-agent-faiss:$(TAG) . \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg MAINTAINER=$(MAINTAINER) + .PHONY: docker/name/agent-sidecar docker/name/agent-sidecar: @echo "$(ORG)/$(AGENT_SIDECAR_IMAGE)" diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index f8303437885..4a02a56a77a 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -19,6 +19,16 @@ e2e: $(call run-e2e-crud-test,-run TestE2EStandardCRUD) +.PHONY: e2e/faiss +## run e2e/faiss +e2e/faiss: + #$(call run-e2e-crud-faiss-test,-run TestE2EInsertOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2ESearchOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2EUpdateOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2ERemoveOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2EInsertAndSearch) + $(call run-e2e-crud-faiss-test,-run TestE2EStandardCRUD) + .PHONY: e2e/skip ## run e2e with skip exists operation e2e/skip: diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index d1ceb3bd6a4..a30bcaf89c6 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -91,6 +91,28 @@ define run-e2e-crud-test -kubeconfig=$(KUBECONFIG) endef +define run-e2e-crud-faiss-test + go test \ + -race \ + -mod=readonly \ + $1 \ + -v $(ROOTDIR)/tests/e2e/crud/crud_faiss_test.go \ + -tags "e2e" \ + -timeout $(E2E_TIMEOUT) \ + -host=$(E2E_BIND_HOST) \ + -port=$(E2E_BIND_PORT) \ + -dataset=$(ROOTDIR)/hack/benchmark/assets/dataset/$(E2E_DATASET_NAME).hdf5 \ + -insert-num=$(E2E_INSERT_COUNT) \ + -search-num=$(E2E_SEARCH_COUNT) \ + -update-num=$(E2E_UPDATE_COUNT) \ + -remove-num=$(E2E_REMOVE_COUNT) \ + -wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \ + -portforward=$(E2E_PORTFORWARD_ENABLED) \ + -portforward-pod-name=$(E2E_TARGET_POD_NAME) \ + -portforward-pod-port=$(E2E_TARGET_PORT) \ + -namespace=$(E2E_TARGET_NAMESPACE) +endef + define run-e2e-multi-crud-test GOPRIVATE=$(GOPRIVATE) \ go test \ @@ -206,3 +228,13 @@ define gen-go-option-test-sources fi; \ done endef + +define gen-crd-schema + @[ -f $(ROOTDIR)/charts/vald-helm-operator/crds/$1.yaml ] \ + && mv $(ROOTDIR)/charts/vald-helm-operator/crds/$1.yaml $(TEMP_DIR)/$1.yaml || true + GOPRIVATE=$(GOPRIVATE) \ + go run -mod=readonly $(ROOTDIR)/hack/helm/schema/crd/main.go \ + $(ROOTDIR)/charts/$2/values.yaml > $(TEMP_DIR)/$1-spec.yaml + $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ + $(TEMP_DIR)/$1.yaml $(TEMP_DIR)/$1-spec.yaml > $(ROOTDIR)/charts/vald-helm-operator/crds/$1.yaml +endef diff --git a/Makefile.d/helm.mk b/Makefile.d/helm.mk index 9d69b9063e2..c4c4171f112 100644 --- a/Makefile.d/helm.mk +++ b/Makefile.d/helm.mk @@ -104,20 +104,10 @@ $(BINDIR)/yq: ## generate OpenAPI v3 schema for ValdRelease helm/schema/crd/vald: \ yq/install - mv charts/vald-helm-operator/crds/valdrelease.yaml $(TEMP_DIR)/valdrelease.yaml - GOPRIVATE=$(GOPRIVATE) \ - go run -mod=readonly hack/helm/schema/crd/main.go \ - charts/vald/values.yaml > $(TEMP_DIR)/valdrelease-spec.yaml - $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ - $(TEMP_DIR)/valdrelease.yaml $(TEMP_DIR)/valdrelease-spec.yaml > charts/vald-helm-operator/crds/valdrelease.yaml + $(call gen-crd-schema,valdrelease,vald) .PHONY: helm/schema/crd/vald-helm-operator ## generate OpenAPI v3 schema for ValdHelmOperatorRelease helm/schema/crd/vald-helm-operator: \ yq/install - mv charts/vald-helm-operator/crds/valdhelmoperatorrelease.yaml $(TEMP_DIR)/valdhelmoperatorrelease.yaml - GOPRIVATE=$(GOPRIVATE) \ - go run -mod=readonly hack/helm/schema/crd/main.go \ - charts/vald-helm-operator/values.yaml > $(TEMP_DIR)/valdhelmoperatorrelease-spec.yaml - $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ - $(TEMP_DIR)/valdhelmoperatorrelease.yaml $(TEMP_DIR)/valdhelmoperatorrelease-spec.yaml > charts/vald-helm-operator/crds/valdhelmoperatorrelease.yaml + $(call gen-crd-schema,valdhelmoperatorrelease,vald-helm-operator) diff --git a/Makefile.d/k3d.mk b/Makefile.d/k3d.mk index 5bda1e19c8b..0977f67575f 100644 --- a/Makefile.d/k3d.mk +++ b/Makefile.d/k3d.mk @@ -30,10 +30,25 @@ $(BINDIR)/k3d: .PHONY: k3d/start ## start k3d (kubernetes in docker) cluster k3d/start: - $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) --image docker.io/rancher/k3s:latest -v "/lib/modules:/lib/modules" - # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) -v "/lib/modules:/lib/modules" --host-pid-mode=true + $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) \ + --agents $(K3D_NODES) \ + --image docker.io/rancher/k3s:latest \ + --host-pid-mode=true \ + --port 8081:80@loadbalancer \ + --k3s-arg "--disable=traefik@server:*" \ + -v "/lib/modules:/lib/modules" + # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) -v "/lib/modules:/lib/modules" # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) -p "8081:80@loadbalancer" --agents $(K3D_NODES) --k3s-arg '--disable=traefik@all' +<<<<<<< HEAD export KUBECONFIG="$(shell sudo $(K3D_COMMAND) kubeconfig merge -o $(TEMP_DIR)/k3d_$(K3D_CLUSTER_NAME)_kubeconfig.yaml $(K3D_CLUSTER_NAME))" +======= + # K3D_KUBECONFIG_PATH="$(TEMP_DIR)/k3d_$(K3D_CLUSTER_NAME)_kubeconfig.yaml" + # $(K3D_COMMAND) kubeconfig merge -o $(K3D_KUBECONFIG_PATH) $(K3D_CLUSTER_NAME) + # export KUBECONFIG="$(K3D_KUBECONFIG_PATH)" + kubectl wait -n kube-system --for=condition=ready pod -l k8s-app=metrics-server --timeout=600s + kubectl apply -f https://projectcontour.io/quickstart/operator.yaml + kubectl apply -f https://projectcontour.io/quickstart/contour-custom-resource.yaml +>>>>>>> feature/agent/new-agents-helm-template-base .PHONY: k3d/restart ## restart k3d (kubernetes in docker) cluster diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index bf0428da583..3b724a44894 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -43,10 +43,20 @@ k8s/manifest/update: \ mv $(TEMP_DIR)/vald/templates/discoverer k8s/discoverer mv $(TEMP_DIR)/vald/templates/gateway/lb k8s/gateway/lb mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index +<<<<<<< HEAD mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction mv $(TEMP_DIR)/vald/templates/index/job/creation k8s/index/job/creation mv $(TEMP_DIR)/vald/templates/index/job/save k8s/index/job/save mv $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate k8s/index/job/readreplica/rotate +======= + # yq ".data.config*yaml" k8s/agent/qbg/configmap.yaml | sed 1d > cmd/agent/core/qbg/sample.yaml + yq ".data.config*yaml" k8s/agent/faiss/configmap.yaml | sed 1d > cmd/agent/core/faiss/sample.yaml + yq ".data.config*yaml" k8s/agent/ngt/configmap.yaml | sed 1d > cmd/agent/core/ngt/sample.yaml + yq ".data.config*yaml" k8s/discoverer/configmap.yaml | sed 1d > cmd/discoverer/k8s/sample.yaml + yq ".data.config*yaml" k8s/gateway/lb/configmap.yaml | sed 1d > cmd/gateway/lb/sample.yaml + yq ".data.config*yaml" k8s/manager/index/configmap.yaml | sed 1d > cmd/manager/index/sample.yaml + yq ".data.config*yaml" k8s/tools/cli/loadtest/configmap.yaml | sed 1d > cmd/tools/cli/loadtest/sample.yaml +>>>>>>> feature/agent/new-agents-helm-template-base rm -rf $(TEMP_DIR) .PHONY: k8s/manifest/helm-operator/clean diff --git a/charts/vald-helm-operator/crds/valdrelease.yaml b/charts/vald-helm-operator/crds/valdrelease.yaml index 08832a86b2f..3587c314a46 100644 --- a/charts/vald-helm-operator/crds/valdrelease.yaml +++ b/charts/vald-helm-operator/crds/valdrelease.yaml @@ -116,6 +116,12 @@ spec: items: type: object x-kubernetes-preserve-unknown-fields: true + algorithm: + type: string + enum: + - ngt + - qbg + - faiss annotations: type: object x-kubernetes-preserve-unknown-fields: true @@ -317,7 +323,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -758,6 +764,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -1248,7 +1256,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -1618,6 +1626,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2098,7 +2108,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -2459,6 +2469,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2918,7 +2930,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -3316,6 +3328,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -4279,7 +4293,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -4677,6 +4691,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -5438,7 +5454,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -5836,6 +5852,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -8518,7 +8536,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -10521,6 +10539,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: diff --git a/charts/vald/README.md b/charts/vald/README.md index 831aba25dc1..0462e7d0496 100644 --- a/charts/vald/README.md +++ b/charts/vald/README.md @@ -44,6 +44,7 @@ Run the following command to install the chart, ### Parameters +<<<<<<< HEAD | Key | Type | Default | Description | | ----------------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | agent.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | @@ -849,3 +850,714 @@ Run the following command to install the chart, | manager.index.version | string | `"v0.0.0"` | version of index manager config | | manager.index.volumeMounts | list | `[]` | volume mounts | | manager.index.volumes | list | `[]` | volumes | +======= +| Key | Type | Default | Description | +| ----------------------------------------------------------------------------------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| agent.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | +| agent.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | +| agent.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | +| agent.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity required scheduling terms | +| agent.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["vald-agent"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | pod anti-affinity preferred scheduling terms | +| agent.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | +| agent.algorithm | string | `"ngt"` | agent algorithm type. it should be `ngt` or `qbg` or `faiss`. | +| agent.annotations | object | `{}` | deployment annotations | +| agent.enabled | bool | `true` | agent enabled | +| agent.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | +| agent.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| agent.hpa.enabled | bool | `false` | HPA enabled | +| agent.hpa.targetCPUUtilizationPercentage | int | `80` | HPA CPU utilization percentage | +| agent.image.pullPolicy | string | `"Always"` | image pull policy | +| agent.image.repository | string | `"vdaas/vald-agent-ngt"` | image repository | +| agent.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| agent.initContainers | list | `[]` | init containers | +| agent.kind | string | `"StatefulSet"` | deployment kind: Deployment, DaemonSet or StatefulSet | +| agent.logging | object | `{}` | logging config (overrides defaults.logging) | +| agent.maxReplicas | int | `300` | maximum number of replicas. if HPA is disabled, this value will be ignored. | +| agent.maxUnavailable | string | `"1"` | maximum number of unavailable replicas | +| agent.minReplicas | int | `20` | minimum number of replicas. if HPA is disabled, the replicas will be set to this value | +| agent.name | string | `"vald-agent"` | name of agent deployment | +| agent.ngt.auto_create_index_pool_size | int | `10000` | batch process pool size of automatic create index operation | +| agent.ngt.auto_index_check_duration | string | `"30m"` | check duration of automatic indexing | +| agent.ngt.auto_index_duration_limit | string | `"24h"` | limit duration of automatic indexing | +| agent.ngt.auto_index_length | int | `100` | number of cache to trigger automatic indexing | +| agent.ngt.auto_save_index_duration | string | `"35m"` | duration of automatic save index | +| agent.ngt.broken_index_history_limit | int | `0` | maximum number of broken index generations to backup | +| agent.ngt.bulk_insert_chunk_size | int | `10` | bulk insert chunk size | +| agent.ngt.creation_edge_size | int | `20` | creation edge size | +| agent.ngt.default_epsilon | float | `0.1` | default epsilon used for search | +| agent.ngt.default_pool_size | int | `10000` | default create index batch pool size | +| agent.ngt.default_radius | float | `-1` | default radius used for search | +| agent.ngt.dimension | int | `4096` | vector dimension | +| agent.ngt.distance_type | string | `"l2"` | distance type. it should be `l1`, `l2`, `angle`, `hamming`, `cosine`,`poincare`, `lorentz`, `jaccard`, `sparsejaccard`, `normalizedangle` or `normalizedcosine`. for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/algorithm/ngt#pkg-constants | +| agent.ngt.enable_copy_on_write | bool | `false` | enable copy on write saving for more stable backup | +| agent.ngt.enable_in_memory_mode | bool | `true` | in-memory mode enabled | +| agent.ngt.enable_proactive_gc | bool | `false` | enable proactive GC call for reducing heap memory allocation | +| agent.ngt.index_path | string | `""` | path to index data | +| agent.ngt.initial_delay_max_duration | string | `"3m"` | maximum duration for initial delay | +| agent.ngt.kvsdb.concurrency | int | `6` | kvsdb processing concurrency | +| agent.ngt.load_index_timeout_factor | string | `"1ms"` | a factor of load index timeout. timeout duration will be calculated by (index count to be loaded) \* (factor). | +| agent.ngt.max_load_index_timeout | string | `"10m"` | maximum duration of load index timeout | +| agent.ngt.min_load_index_timeout | string | `"3m"` | minimum duration of load index timeout | +| agent.ngt.object_type | string | `"float"` | object type. it should be `float` or `uint8` or `float16`. for further details: https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference | +| agent.ngt.search_edge_size | int | `10` | search edge size | +| agent.ngt.vqueue.delete_buffer_pool_size | int | `5000` | delete slice pool buffer size | +| agent.ngt.vqueue.insert_buffer_pool_size | int | `10000` | insert slice pool buffer size | +| agent.nodeName | string | `""` | node name | +| agent.nodeSelector | object | `{}` | node selector | +| agent.observability | object | `{"otlp":{"attribute":{"service_name":"vald-agent"}}}` | observability config (overrides defaults.observability) | +| agent.persistentVolume.accessMode | string | `"ReadWriteOncePod"` | agent pod storage accessMode | +| agent.persistentVolume.enabled | bool | `false` | enables PVC. It is required to enable if agent pod's file store functionality is enabled with non in-memory mode | +| agent.persistentVolume.mountPropagation | string | `"None"` | agent pod storage mountPropagation | +| agent.persistentVolume.size | string | `"100Gi"` | size of agent pod volume | +| agent.persistentVolume.storageClass | string | `"vald-sc"` | storageClass name for agent pod volume | +| agent.podAnnotations | object | `{}` | pod annotations | +| agent.podManagementPolicy | string | `"OrderedReady"` | pod management policy: OrderedReady or Parallel | +| agent.podPriority.enabled | bool | `true` | agent pod PriorityClass enabled | +| agent.podPriority.value | int | `1000000000` | agent pod PriorityClass value | +| agent.podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | +| agent.progressDeadlineSeconds | int | `600` | progress deadline seconds | +| agent.resources | object | `{"requests":{"cpu":"300m","memory":"4Gi"}}` | compute resources. recommended setting of memory requests = cluster memory \* 0.4 / number of agent pods | +| agent.revisionHistoryLimit | int | `2` | number of old history to retain to allow rollback | +| agent.rollingUpdate.maxSurge | string | `"25%"` | max surge of rolling update | +| agent.rollingUpdate.maxUnavailable | string | `"25%"` | max unavailable of rolling update | +| agent.rollingUpdate.partition | int | `0` | StatefulSet partition | +| agent.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":false,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for container | +| agent.server_config | object | `{"healths":{"liveness":{},"readiness":{},"startup":{"startupProbe":{"failureThreshold":200,"periodSeconds":5}}},"metrics":{"pprof":{}},"servers":{"grpc":{},"rest":{}}}` | server config (overrides defaults.server_config) | +| agent.service.annotations | object | `{}` | service annotations | +| agent.service.labels | object | `{}` | service labels | +| agent.serviceType | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| agent.sidecar.config.auto_backup_duration | string | `"24h"` | auto backup duration | +| agent.sidecar.config.auto_backup_enabled | bool | `true` | auto backup triggered by timer is enabled | +| agent.sidecar.config.blob_storage.bucket | string | `""` | bucket name | +| agent.sidecar.config.blob_storage.cloud_storage.client.credentials_file_path | string | `""` | credentials file path | +| agent.sidecar.config.blob_storage.cloud_storage.client.credentials_json | string | `""` | credentials json | +| agent.sidecar.config.blob_storage.cloud_storage.url | string | `""` | cloud storage url | +| agent.sidecar.config.blob_storage.cloud_storage.write_buffer_size | int | `0` | bytes of the chunks for upload | +| agent.sidecar.config.blob_storage.cloud_storage.write_cache_control | string | `""` | Cache-Control of HTTP Header | +| agent.sidecar.config.blob_storage.cloud_storage.write_content_disposition | string | `""` | Content-Disposition of HTTP Header | +| agent.sidecar.config.blob_storage.cloud_storage.write_content_encoding | string | `""` | the encoding of the blob's content | +| agent.sidecar.config.blob_storage.cloud_storage.write_content_language | string | `""` | the language of blob's content | +| agent.sidecar.config.blob_storage.cloud_storage.write_content_type | string | `""` | MIME type of the blob | +| agent.sidecar.config.blob_storage.s3.access_key | string | `"_AWS_ACCESS_KEY_"` | s3 access key | +| agent.sidecar.config.blob_storage.s3.enable_100_continue | bool | `true` | enable AWS SDK adding the 'Expect: 100-Continue' header to PUT requests over 2MB of content. | +| agent.sidecar.config.blob_storage.s3.enable_content_md5_validation | bool | `true` | enable the S3 client to add MD5 checksum to upload API calls. | +| agent.sidecar.config.blob_storage.s3.enable_endpoint_discovery | bool | `false` | enable endpoint discovery | +| agent.sidecar.config.blob_storage.s3.enable_endpoint_host_prefix | bool | `true` | enable prefixing request endpoint hosts with modeled information | +| agent.sidecar.config.blob_storage.s3.enable_param_validation | bool | `true` | enables semantic parameter validation | +| agent.sidecar.config.blob_storage.s3.enable_ssl | bool | `true` | enable ssl for s3 session | +| agent.sidecar.config.blob_storage.s3.endpoint | string | `""` | s3 endpoint | +| agent.sidecar.config.blob_storage.s3.force_path_style | bool | `false` | use path-style addressing | +| agent.sidecar.config.blob_storage.s3.max_chunk_size | string | `"64mb"` | s3 download max chunk size | +| agent.sidecar.config.blob_storage.s3.max_part_size | string | `"64mb"` | s3 multipart upload max part size | +| agent.sidecar.config.blob_storage.s3.max_retries | int | `3` | maximum number of retries of s3 client | +| agent.sidecar.config.blob_storage.s3.region | string | `""` | s3 region | +| agent.sidecar.config.blob_storage.s3.secret_access_key | string | `"_AWS_SECRET_ACCESS_KEY_"` | s3 secret access key | +| agent.sidecar.config.blob_storage.s3.token | string | `""` | s3 token | +| agent.sidecar.config.blob_storage.s3.use_accelerate | bool | `false` | enable s3 accelerate feature | +| agent.sidecar.config.blob_storage.s3.use_arn_region | bool | `false` | s3 service client to use the region specified in the ARN | +| agent.sidecar.config.blob_storage.s3.use_dual_stack | bool | `false` | use dual stack | +| agent.sidecar.config.blob_storage.storage_type | string | `"s3"` | storage type | +| agent.sidecar.config.client.net.dialer.dual_stack_enabled | bool | `false` | HTTP client TCP dialer dual stack enabled | +| agent.sidecar.config.client.net.dialer.keepalive | string | `"5m"` | HTTP client TCP dialer keep alive | +| agent.sidecar.config.client.net.dialer.timeout | string | `"5s"` | HTTP client TCP dialer connect timeout | +| agent.sidecar.config.client.net.dns.cache_enabled | bool | `true` | HTTP client TCP DNS cache enabled | +| agent.sidecar.config.client.net.dns.cache_expiration | string | `"24h"` | | +| agent.sidecar.config.client.net.dns.refresh_duration | string | `"1h"` | HTTP client TCP DNS cache expiration | +| agent.sidecar.config.client.net.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| agent.sidecar.config.client.net.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| agent.sidecar.config.client.net.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| agent.sidecar.config.client.net.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| agent.sidecar.config.client.net.socket_option.tcp_cork | bool | `true` | server listen socket option for tcp_cork functionality | +| agent.sidecar.config.client.net.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| agent.sidecar.config.client.net.socket_option.tcp_fast_open | bool | `true` | server listen socket option for tcp_fast_open functionality | +| agent.sidecar.config.client.net.socket_option.tcp_no_delay | bool | `false` | server listen socket option for tcp_no_delay functionality | +| agent.sidecar.config.client.net.socket_option.tcp_quick_ack | bool | `false` | server listen socket option for tcp_quick_ack functionality | +| agent.sidecar.config.client.net.tls.ca | string | `"/path/to/ca"` | TLS ca path | +| agent.sidecar.config.client.net.tls.cert | string | `"/path/to/cert"` | TLS cert path | +| agent.sidecar.config.client.net.tls.enabled | bool | `false` | TLS enabled | +| agent.sidecar.config.client.net.tls.insecure_skip_verify | bool | `false` | enable/disable skip SSL certificate verification | +| agent.sidecar.config.client.net.tls.key | string | `"/path/to/key"` | TLS key path | +| agent.sidecar.config.client.transport.backoff.backoff_factor | float | `1.1` | backoff backoff factor | +| agent.sidecar.config.client.transport.backoff.backoff_time_limit | string | `"5s"` | backoff time limit | +| agent.sidecar.config.client.transport.backoff.enable_error_log | bool | `true` | backoff error log enabled | +| agent.sidecar.config.client.transport.backoff.initial_duration | string | `"5ms"` | backoff initial duration | +| agent.sidecar.config.client.transport.backoff.jitter_limit | string | `"100ms"` | backoff jitter limit | +| agent.sidecar.config.client.transport.backoff.maximum_duration | string | `"5s"` | backoff maximum duration | +| agent.sidecar.config.client.transport.backoff.retry_count | int | `100` | backoff retry count | +| agent.sidecar.config.client.transport.round_tripper.expect_continue_timeout | string | `"5s"` | expect continue timeout | +| agent.sidecar.config.client.transport.round_tripper.force_attempt_http_2 | bool | `true` | force attempt HTTP2 | +| agent.sidecar.config.client.transport.round_tripper.idle_conn_timeout | string | `"90s"` | timeout for idle connections | +| agent.sidecar.config.client.transport.round_tripper.max_conns_per_host | int | `10` | maximum count of connections per host | +| agent.sidecar.config.client.transport.round_tripper.max_idle_conns | int | `100` | maximum count of idle connections | +| agent.sidecar.config.client.transport.round_tripper.max_idle_conns_per_host | int | `10` | maximum count of idle connections per host | +| agent.sidecar.config.client.transport.round_tripper.max_response_header_size | int | `0` | maximum response header size | +| agent.sidecar.config.client.transport.round_tripper.read_buffer_size | int | `0` | read buffer size | +| agent.sidecar.config.client.transport.round_tripper.response_header_timeout | string | `"5s"` | timeout for response header | +| agent.sidecar.config.client.transport.round_tripper.tls_handshake_timeout | string | `"5s"` | TLS handshake timeout | +| agent.sidecar.config.client.transport.round_tripper.write_buffer_size | int | `0` | write buffer size | +| agent.sidecar.config.compress.compress_algorithm | string | `"gzip"` | compression algorithm. must be `gob`, `gzip`, `lz4` or `zstd` | +| agent.sidecar.config.compress.compression_level | int | `-1` | compression level. value range relies on which algorithm is used. `gob`: level will be ignored. `gzip`: -1 (default compression), 0 (no compression), or 1 (best speed) to 9 (best compression). `lz4`: >= 0, higher is better compression. `zstd`: 1 (fastest) to 22 (best), however implementation relies on klauspost/compress. | +| agent.sidecar.config.filename | string | `"_MY_POD_NAME_"` | backup filename | +| agent.sidecar.config.filename_suffix | string | `".tar.gz"` | suffix for backup filename | +| agent.sidecar.config.post_stop_timeout | string | `"2m"` | timeout for observing file changes during post stop | +| agent.sidecar.config.restore_backoff.backoff_factor | float | `1.2` | restore backoff factor | +| agent.sidecar.config.restore_backoff.backoff_time_limit | string | `"30m"` | restore backoff time limit | +| agent.sidecar.config.restore_backoff.enable_error_log | bool | `true` | restore backoff log enabled | +| agent.sidecar.config.restore_backoff.initial_duration | string | `"1s"` | restore backoff initial duration | +| agent.sidecar.config.restore_backoff.jitter_limit | string | `"10s"` | restore backoff jitter limit | +| agent.sidecar.config.restore_backoff.maximum_duration | string | `"1m"` | restore backoff maximum duration | +| agent.sidecar.config.restore_backoff.retry_count | int | `100` | restore backoff retry count | +| agent.sidecar.config.restore_backoff_enabled | bool | `false` | restore backoff enabled | +| agent.sidecar.config.watch_enabled | bool | `true` | auto backup triggered by file changes is enabled | +| agent.sidecar.enabled | bool | `false` | sidecar enabled | +| agent.sidecar.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}},{"name":"AWS_ACCESS_KEY","valueFrom":{"secretKeyRef":{"key":"access-key","name":"aws-secret"}}},{"name":"AWS_SECRET_ACCESS_KEY","valueFrom":{"secretKeyRef":{"key":"secret-access-key","name":"aws-secret"}}}]` | environment variables | +| agent.sidecar.image.pullPolicy | string | `"Always"` | image pull policy | +| agent.sidecar.image.repository | string | `"vdaas/vald-agent-sidecar"` | image repository | +| agent.sidecar.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| agent.sidecar.initContainerEnabled | bool | `false` | sidecar on initContainer mode enabled. | +| agent.sidecar.logging | object | `{}` | logging config (overrides defaults.logging) | +| agent.sidecar.name | string | `"vald-agent-sidecar"` | name of agent sidecar | +| agent.sidecar.observability | object | `{"otlp":{"attribute":{"service_name":"vald-agent-sidecar"}}}` | observability config (overrides defaults.observability) | +| agent.sidecar.resources | object | `{"requests":{"cpu":"100m","memory":"100Mi"}}` | compute resources. | +| agent.sidecar.server_config | object | `{"healths":{"liveness":{"enabled":false,"port":13000,"servicePort":13000},"readiness":{"enabled":false,"port":13001,"servicePort":13001},"startup":{"enabled":false,"port":13001}},"metrics":{"pprof":{"port":16060,"servicePort":16060}},"servers":{"grpc":{"enabled":false,"port":18081,"servicePort":18081},"rest":{"enabled":false,"port":18080,"servicePort":18080}}}` | server config (overrides defaults.server_config) | +| agent.sidecar.service.annotations | object | `{}` | agent sidecar service annotations | +| agent.sidecar.service.enabled | bool | `false` | agent sidecar service enabled | +| agent.sidecar.service.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| agent.sidecar.service.labels | object | `{}` | agent sidecar service labels | +| agent.sidecar.service.type | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| agent.sidecar.time_zone | string | `""` | Time zone | +| agent.sidecar.version | string | `"v0.0.0"` | version of agent sidecar config | +| agent.terminationGracePeriodSeconds | int | `120` | duration in seconds pod needs to terminate gracefully | +| agent.time_zone | string | `""` | Time zone | +| agent.tolerations | list | `[]` | tolerations | +| agent.topologySpreadConstraints | list | `[]` | topology spread constraints for agent pods | +| agent.version | string | `"v0.0.0"` | version of agent config | +| agent.volumeMounts | list | `[]` | volume mounts | +| agent.volumes | list | `[]` | volumes | +| defaults.grpc.client.addrs | list | `[]` | gRPC client addresses | +| defaults.grpc.client.backoff.backoff_factor | float | `1.1` | gRPC client backoff factor | +| defaults.grpc.client.backoff.backoff_time_limit | string | `"5s"` | gRPC client backoff time limit | +| defaults.grpc.client.backoff.enable_error_log | bool | `true` | gRPC client backoff log enabled | +| defaults.grpc.client.backoff.initial_duration | string | `"5ms"` | gRPC client backoff initial duration | +| defaults.grpc.client.backoff.jitter_limit | string | `"100ms"` | gRPC client backoff jitter limit | +| defaults.grpc.client.backoff.maximum_duration | string | `"5s"` | gRPC client backoff maximum duration | +| defaults.grpc.client.backoff.retry_count | int | `100` | gRPC client backoff retry count | +| defaults.grpc.client.call_option.max_recv_msg_size | int | `0` | gRPC client call option max receive message size | +| defaults.grpc.client.call_option.max_retry_rpc_buffer_size | int | `0` | gRPC client call option max retry rpc buffer size | +| defaults.grpc.client.call_option.max_send_msg_size | int | `0` | gRPC client call option max send message size | +| defaults.grpc.client.call_option.wait_for_ready | bool | `true` | gRPC client call option wait for ready | +| defaults.grpc.client.circuit_breaker.closed_error_rate | float | `0.7` | gRPC client circuitbreaker closed error rate | +| defaults.grpc.client.circuit_breaker.closed_refresh_timeout | string | `"10s"` | gRPC client circuitbreaker closed refresh timeout | +| defaults.grpc.client.circuit_breaker.half_open_error_rate | float | `0.5` | gRPC client circuitbreaker half-open error rate | +| defaults.grpc.client.circuit_breaker.min_samples | int | `1000` | gRPC client circuitbreaker minimum sampling count | +| defaults.grpc.client.circuit_breaker.open_timeout | string | `"1s"` | gRPC client circuitbreaker open timeout | +| defaults.grpc.client.connection_pool.enable_dns_resolver | bool | `true` | enables gRPC client connection pool dns resolver, when enabled vald uses ip handshake exclude dns discovery which improves network performance | +| defaults.grpc.client.connection_pool.enable_rebalance | bool | `true` | enables gRPC client connection pool rebalance | +| defaults.grpc.client.connection_pool.old_conn_close_duration | string | `"2m"` | makes delay before gRPC client connection closing during connection pool rebalance | +| defaults.grpc.client.connection_pool.rebalance_duration | string | `"30m"` | gRPC client connection pool rebalance duration | +| defaults.grpc.client.connection_pool.size | int | `3` | gRPC client connection pool size | +| defaults.grpc.client.dial_option.backoff_base_delay | string | `"1s"` | gRPC client dial option base backoff delay | +| defaults.grpc.client.dial_option.backoff_jitter | float | `0.2` | gRPC client dial option base backoff delay | +| defaults.grpc.client.dial_option.backoff_max_delay | string | `"120s"` | gRPC client dial option max backoff delay | +| defaults.grpc.client.dial_option.backoff_multiplier | float | `1.6` | gRPC client dial option base backoff delay | +| defaults.grpc.client.dial_option.enable_backoff | bool | `false` | gRPC client dial option backoff enabled | +| defaults.grpc.client.dial_option.initial_connection_window_size | int | `2097152` | gRPC client dial option initial connection window size | +| defaults.grpc.client.dial_option.initial_window_size | int | `1048576` | gRPC client dial option initial window size | +| defaults.grpc.client.dial_option.insecure | bool | `true` | gRPC client dial option insecure enabled | +| defaults.grpc.client.dial_option.interceptors | list | `[]` | gRPC client interceptors | +| defaults.grpc.client.dial_option.keepalive.permit_without_stream | bool | `false` | gRPC client keep alive permit without stream | +| defaults.grpc.client.dial_option.keepalive.time | string | `""` | gRPC client keep alive time | +| defaults.grpc.client.dial_option.keepalive.timeout | string | `"30s"` | gRPC client keep alive timeout | +| defaults.grpc.client.dial_option.max_msg_size | int | `0` | gRPC client dial option max message size | +| defaults.grpc.client.dial_option.min_connection_timeout | string | `"20s"` | gRPC client dial option minimum connection timeout | +| defaults.grpc.client.dial_option.net.dialer.dual_stack_enabled | bool | `true` | gRPC client TCP dialer dual stack enabled | +| defaults.grpc.client.dial_option.net.dialer.keepalive | string | `""` | gRPC client TCP dialer keep alive | +| defaults.grpc.client.dial_option.net.dialer.timeout | string | `""` | gRPC client TCP dialer timeout | +| defaults.grpc.client.dial_option.net.dns.cache_enabled | bool | `true` | gRPC client TCP DNS cache enabled | +| defaults.grpc.client.dial_option.net.dns.cache_expiration | string | `"1h"` | gRPC client TCP DNS cache expiration | +| defaults.grpc.client.dial_option.net.dns.refresh_duration | string | `"30m"` | gRPC client TCP DNS cache refresh duration | +| defaults.grpc.client.dial_option.net.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| defaults.grpc.client.dial_option.net.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| defaults.grpc.client.dial_option.net.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| defaults.grpc.client.dial_option.net.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| defaults.grpc.client.dial_option.net.socket_option.tcp_cork | bool | `false` | server listen socket option for tcp_cork functionality | +| defaults.grpc.client.dial_option.net.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| defaults.grpc.client.dial_option.net.socket_option.tcp_fast_open | bool | `false` | server listen socket option for tcp_fast_open functionality | +| defaults.grpc.client.dial_option.net.socket_option.tcp_no_delay | bool | `false` | server listen socket option for tcp_no_delay functionality | +| defaults.grpc.client.dial_option.net.socket_option.tcp_quick_ack | bool | `false` | server listen socket option for tcp_quick_ack functionality | +| defaults.grpc.client.dial_option.net.tls.ca | string | `"/path/to/ca"` | TLS ca path | +| defaults.grpc.client.dial_option.net.tls.cert | string | `"/path/to/cert"` | TLS cert path | +| defaults.grpc.client.dial_option.net.tls.enabled | bool | `false` | TLS enabled | +| defaults.grpc.client.dial_option.net.tls.insecure_skip_verify | bool | `false` | enable/disable skip SSL certificate verification | +| defaults.grpc.client.dial_option.net.tls.key | string | `"/path/to/key"` | TLS key path | +| defaults.grpc.client.dial_option.read_buffer_size | int | `0` | gRPC client dial option read buffer size | +| defaults.grpc.client.dial_option.timeout | string | `""` | gRPC client dial option timeout | +| defaults.grpc.client.dial_option.write_buffer_size | int | `0` | gRPC client dial option write buffer size | +| defaults.grpc.client.health_check_duration | string | `"1s"` | gRPC client health check duration | +| defaults.grpc.client.tls.ca | string | `"/path/to/ca"` | TLS ca path | +| defaults.grpc.client.tls.cert | string | `"/path/to/cert"` | TLS cert path | +| defaults.grpc.client.tls.enabled | bool | `false` | TLS enabled | +| defaults.grpc.client.tls.insecure_skip_verify | bool | `false` | enable/disable skip SSL certificate verification | +| defaults.grpc.client.tls.key | string | `"/path/to/key"` | TLS key path | +| defaults.image.tag | string | `"v1.7.6"` | docker image tag | +| defaults.logging.format | string | `"raw"` | logging format. logging format must be `raw` or `json` | +| defaults.logging.level | string | `"debug"` | logging level. logging level must be `debug`, `info`, `warn`, `error` or `fatal`. | +| defaults.logging.logger | string | `"glg"` | logger name. currently logger must be `glg` or `zap`. | +| defaults.networkPolicy.custom | object | `{"egress":{},"ingress":{}}` | custom network policies that a user can add | +| defaults.networkPolicy.custom.egress | object | `{}` | custom egress network policies that a user can add | +| defaults.networkPolicy.custom.ingress | object | `{}` | custom ingress network policies that a user can add | +| defaults.networkPolicy.enabled | bool | `false` | if network policy enabled | +| defaults.observability.enabled | bool | `false` | observability features enabled | +| defaults.observability.metrics.enable_cgo | bool | `true` | CGO metrics enabled | +| defaults.observability.metrics.enable_goroutine | bool | `true` | goroutine metrics enabled | +| defaults.observability.metrics.enable_memory | bool | `true` | memory metrics enabled | +| defaults.observability.metrics.enable_version_info | bool | `true` | version info metrics enabled | +| defaults.observability.metrics.version_info_labels | list | `["vald_version","server_name","git_commit","build_time","go_version","go_os","go_arch","algorithm_info"]` | enabled label names of version info | +| defaults.observability.otlp.attribute | object | `{"namespace":"_MY_POD_NAMESPACE_","node_name":"_MY_NODE_NAME_","pod_name":"_MY_POD_NAME_","service_name":"vald"}` | default resource attribute | +| defaults.observability.otlp.attribute.namespace | string | `"_MY_POD_NAMESPACE_"` | namespace | +| defaults.observability.otlp.attribute.node_name | string | `"_MY_NODE_NAME_"` | node name | +| defaults.observability.otlp.attribute.pod_name | string | `"_MY_POD_NAME_"` | pod name | +| defaults.observability.otlp.attribute.service_name | string | `"vald"` | service name | +| defaults.observability.otlp.collector_endpoint | string | `""` | OpenTelemetry Collector endpoint | +| defaults.observability.otlp.metrics_export_interval | string | `"1s"` | metrics export interval | +| defaults.observability.otlp.metrics_export_timeout | string | `"1m"` | metrics export timeout | +| defaults.observability.otlp.trace_batch_timeout | string | `"1s"` | trace batch timeout | +| defaults.observability.otlp.trace_export_timeout | string | `"1m"` | trace export timeout | +| defaults.observability.otlp.trace_max_export_batch_size | int | `1024` | trace maximum export batch size | +| defaults.observability.otlp.trace_max_queue_size | int | `256` | trace maximum queue size | +| defaults.observability.trace.enabled | bool | `false` | trace enabled | +| defaults.server_config.full_shutdown_duration | string | `"600s"` | server full shutdown duration | +| defaults.server_config.healths.liveness.enabled | bool | `true` | liveness server enabled | +| defaults.server_config.healths.liveness.host | string | `"0.0.0.0"` | liveness server host | +| defaults.server_config.healths.liveness.livenessProbe.failureThreshold | int | `2` | liveness probe failure threshold | +| defaults.server_config.healths.liveness.livenessProbe.httpGet.path | string | `"/liveness"` | liveness probe path | +| defaults.server_config.healths.liveness.livenessProbe.httpGet.port | string | `"liveness"` | liveness probe port | +| defaults.server_config.healths.liveness.livenessProbe.httpGet.scheme | string | `"HTTP"` | liveness probe scheme | +| defaults.server_config.healths.liveness.livenessProbe.initialDelaySeconds | int | `5` | liveness probe initial delay seconds | +| defaults.server_config.healths.liveness.livenessProbe.periodSeconds | int | `3` | liveness probe period seconds | +| defaults.server_config.healths.liveness.livenessProbe.successThreshold | int | `1` | liveness probe success threshold | +| defaults.server_config.healths.liveness.livenessProbe.timeoutSeconds | int | `2` | liveness probe timeout seconds | +| defaults.server_config.healths.liveness.port | int | `3000` | liveness server port | +| defaults.server_config.healths.liveness.server.http.handler_timeout | string | `""` | liveness server handler timeout | +| defaults.server_config.healths.liveness.server.http.idle_timeout | string | `""` | liveness server idle timeout | +| defaults.server_config.healths.liveness.server.http.read_header_timeout | string | `""` | liveness server read header timeout | +| defaults.server_config.healths.liveness.server.http.read_timeout | string | `""` | liveness server read timeout | +| defaults.server_config.healths.liveness.server.http.shutdown_duration | string | `"5s"` | liveness server shutdown duration | +| defaults.server_config.healths.liveness.server.http.write_timeout | string | `""` | liveness server write timeout | +| defaults.server_config.healths.liveness.server.mode | string | `""` | liveness server mode | +| defaults.server_config.healths.liveness.server.network | string | `"tcp"` | mysql network | +| defaults.server_config.healths.liveness.server.probe_wait_time | string | `"3s"` | liveness server probe wait time | +| defaults.server_config.healths.liveness.server.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| defaults.server_config.healths.liveness.server.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| defaults.server_config.healths.liveness.server.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| defaults.server_config.healths.liveness.server.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| defaults.server_config.healths.liveness.server.socket_option.tcp_cork | bool | `false` | server listen socket option for tcp_cork functionality | +| defaults.server_config.healths.liveness.server.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| defaults.server_config.healths.liveness.server.socket_option.tcp_fast_open | bool | `true` | server listen socket option for tcp_fast_open functionality | +| defaults.server_config.healths.liveness.server.socket_option.tcp_no_delay | bool | `true` | server listen socket option for tcp_no_delay functionality | +| defaults.server_config.healths.liveness.server.socket_option.tcp_quick_ack | bool | `true` | server listen socket option for tcp_quick_ack functionality | +| defaults.server_config.healths.liveness.server.socket_path | string | `""` | mysql socket_path | +| defaults.server_config.healths.liveness.servicePort | int | `3000` | liveness server service port | +| defaults.server_config.healths.readiness.enabled | bool | `true` | readiness server enabled | +| defaults.server_config.healths.readiness.host | string | `"0.0.0.0"` | readiness server host | +| defaults.server_config.healths.readiness.port | int | `3001` | readiness server port | +| defaults.server_config.healths.readiness.readinessProbe.failureThreshold | int | `2` | readiness probe failure threshold | +| defaults.server_config.healths.readiness.readinessProbe.httpGet.path | string | `"/readiness"` | readiness probe path | +| defaults.server_config.healths.readiness.readinessProbe.httpGet.port | string | `"readiness"` | readiness probe port | +| defaults.server_config.healths.readiness.readinessProbe.httpGet.scheme | string | `"HTTP"` | readiness probe scheme | +| defaults.server_config.healths.readiness.readinessProbe.initialDelaySeconds | int | `10` | readiness probe initial delay seconds | +| defaults.server_config.healths.readiness.readinessProbe.periodSeconds | int | `3` | readiness probe period seconds | +| defaults.server_config.healths.readiness.readinessProbe.successThreshold | int | `1` | readiness probe success threshold | +| defaults.server_config.healths.readiness.readinessProbe.timeoutSeconds | int | `2` | readiness probe timeout seconds | +| defaults.server_config.healths.readiness.server.http.handler_timeout | string | `""` | readiness server handler timeout | +| defaults.server_config.healths.readiness.server.http.idle_timeout | string | `""` | readiness server idle timeout | +| defaults.server_config.healths.readiness.server.http.read_header_timeout | string | `""` | readiness server read header timeout | +| defaults.server_config.healths.readiness.server.http.read_timeout | string | `""` | readiness server read timeout | +| defaults.server_config.healths.readiness.server.http.shutdown_duration | string | `"0s"` | readiness server shutdown duration | +| defaults.server_config.healths.readiness.server.http.write_timeout | string | `""` | readiness server write timeout | +| defaults.server_config.healths.readiness.server.mode | string | `""` | readiness server mode | +| defaults.server_config.healths.readiness.server.network | string | `"tcp"` | mysql network | +| defaults.server_config.healths.readiness.server.probe_wait_time | string | `"3s"` | readiness server probe wait time | +| defaults.server_config.healths.readiness.server.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| defaults.server_config.healths.readiness.server.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| defaults.server_config.healths.readiness.server.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| defaults.server_config.healths.readiness.server.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| defaults.server_config.healths.readiness.server.socket_option.tcp_cork | bool | `false` | server listen socket option for tcp_cork functionality | +| defaults.server_config.healths.readiness.server.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| defaults.server_config.healths.readiness.server.socket_option.tcp_fast_open | bool | `true` | | +| defaults.server_config.healths.readiness.server.socket_option.tcp_no_delay | bool | `true` | server listen socket option for tcp_no_delay functionality | +| defaults.server_config.healths.readiness.server.socket_option.tcp_quick_ack | bool | `true` | server listen socket option for tcp_quick_ack functionality | +| defaults.server_config.healths.readiness.server.socket_path | string | `""` | mysql socket_path | +| defaults.server_config.healths.readiness.servicePort | int | `3001` | readiness server service port | +| defaults.server_config.healths.startup.enabled | bool | `true` | startup server enabled | +| defaults.server_config.healths.startup.port | int | `3000` | startup server port | +| defaults.server_config.healths.startup.startupProbe.failureThreshold | int | `30` | startup probe failure threshold | +| defaults.server_config.healths.startup.startupProbe.httpGet.path | string | `"/liveness"` | startup probe path | +| defaults.server_config.healths.startup.startupProbe.httpGet.port | string | `"liveness"` | startup probe port | +| defaults.server_config.healths.startup.startupProbe.httpGet.scheme | string | `"HTTP"` | startup probe scheme | +| defaults.server_config.healths.startup.startupProbe.initialDelaySeconds | int | `5` | startup probe initial delay seconds | +| defaults.server_config.healths.startup.startupProbe.periodSeconds | int | `5` | startup probe period seconds | +| defaults.server_config.healths.startup.startupProbe.successThreshold | int | `1` | startup probe success threshold | +| defaults.server_config.healths.startup.startupProbe.timeoutSeconds | int | `2` | startup probe timeout seconds | +| defaults.server_config.metrics.pprof.enabled | bool | `false` | pprof server enabled | +| defaults.server_config.metrics.pprof.host | string | `"0.0.0.0"` | pprof server host | +| defaults.server_config.metrics.pprof.port | int | `6060` | pprof server port | +| defaults.server_config.metrics.pprof.server.http.handler_timeout | string | `"5s"` | pprof server handler timeout | +| defaults.server_config.metrics.pprof.server.http.idle_timeout | string | `"2s"` | pprof server idle timeout | +| defaults.server_config.metrics.pprof.server.http.read_header_timeout | string | `"1s"` | pprof server read header timeout | +| defaults.server_config.metrics.pprof.server.http.read_timeout | string | `"1s"` | pprof server read timeout | +| defaults.server_config.metrics.pprof.server.http.shutdown_duration | string | `"5s"` | pprof server shutdown duration | +| defaults.server_config.metrics.pprof.server.http.write_timeout | string | `"1m"` | pprof server write timeout | +| defaults.server_config.metrics.pprof.server.mode | string | `"REST"` | pprof server mode | +| defaults.server_config.metrics.pprof.server.network | string | `"tcp"` | mysql network | +| defaults.server_config.metrics.pprof.server.probe_wait_time | string | `"3s"` | pprof server probe wait time | +| defaults.server_config.metrics.pprof.server.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| defaults.server_config.metrics.pprof.server.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| defaults.server_config.metrics.pprof.server.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| defaults.server_config.metrics.pprof.server.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| defaults.server_config.metrics.pprof.server.socket_option.tcp_cork | bool | `true` | server listen socket option for tcp_cork functionality | +| defaults.server_config.metrics.pprof.server.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| defaults.server_config.metrics.pprof.server.socket_option.tcp_fast_open | bool | `false` | server listen socket option for tcp_fast_open functionality | +| defaults.server_config.metrics.pprof.server.socket_option.tcp_no_delay | bool | `false` | server listen socket option for tcp_no_delay functionality | +| defaults.server_config.metrics.pprof.server.socket_option.tcp_quick_ack | bool | `false` | server listen socket option for tcp_quick_ack functionality | +| defaults.server_config.metrics.pprof.server.socket_path | string | `""` | mysql socket_path | +| defaults.server_config.metrics.pprof.servicePort | int | `6060` | pprof server service port | +| defaults.server_config.servers.grpc.enabled | bool | `true` | gRPC server enabled | +| defaults.server_config.servers.grpc.host | string | `"0.0.0.0"` | gRPC server host | +| defaults.server_config.servers.grpc.port | int | `8081` | gRPC server port | +| defaults.server_config.servers.grpc.server.grpc.bidirectional_stream_concurrency | int | `20` | gRPC server bidirectional stream concurrency | +| defaults.server_config.servers.grpc.server.grpc.connection_timeout | string | `""` | gRPC server connection timeout | +| defaults.server_config.servers.grpc.server.grpc.enable_admin | bool | `true` | gRPC server admin option | +| defaults.server_config.servers.grpc.server.grpc.enable_reflection | bool | `true` | gRPC server reflection option | +| defaults.server_config.servers.grpc.server.grpc.header_table_size | int | `0` | gRPC server header table size | +| defaults.server_config.servers.grpc.server.grpc.initial_conn_window_size | int | `2097152` | gRPC server initial connection window size | +| defaults.server_config.servers.grpc.server.grpc.initial_window_size | int | `1048576` | gRPC server initial window size | +| defaults.server_config.servers.grpc.server.grpc.interceptors | list | `["RecoverInterceptor"]` | gRPC server interceptors | +| defaults.server_config.servers.grpc.server.grpc.keepalive.max_conn_age | string | `""` | gRPC server keep alive max connection age | +| defaults.server_config.servers.grpc.server.grpc.keepalive.max_conn_age_grace | string | `""` | gRPC server keep alive max connection age grace | +| defaults.server_config.servers.grpc.server.grpc.keepalive.max_conn_idle | string | `""` | gRPC server keep alive max connection idle | +| defaults.server_config.servers.grpc.server.grpc.keepalive.min_time | string | `"10m"` | gRPC server keep alive min_time | +| defaults.server_config.servers.grpc.server.grpc.keepalive.permit_without_stream | bool | `false` | gRPC server keep alive permit_without_stream | +| defaults.server_config.servers.grpc.server.grpc.keepalive.time | string | `"3h"` | gRPC server keep alive time | +| defaults.server_config.servers.grpc.server.grpc.keepalive.timeout | string | `"60s"` | gRPC server keep alive timeout | +| defaults.server_config.servers.grpc.server.grpc.max_header_list_size | int | `0` | gRPC server max header list size | +| defaults.server_config.servers.grpc.server.grpc.max_receive_message_size | int | `0` | gRPC server max receive message size | +| defaults.server_config.servers.grpc.server.grpc.max_send_message_size | int | `0` | gRPC server max send message size | +| defaults.server_config.servers.grpc.server.grpc.read_buffer_size | int | `0` | gRPC server read buffer size | +| defaults.server_config.servers.grpc.server.grpc.write_buffer_size | int | `0` | gRPC server write buffer size | +| defaults.server_config.servers.grpc.server.mode | string | `"GRPC"` | gRPC server server mode | +| defaults.server_config.servers.grpc.server.network | string | `"tcp"` | mysql network | +| defaults.server_config.servers.grpc.server.probe_wait_time | string | `"3s"` | gRPC server probe wait time | +| defaults.server_config.servers.grpc.server.restart | bool | `true` | gRPC server restart | +| defaults.server_config.servers.grpc.server.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| defaults.server_config.servers.grpc.server.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| defaults.server_config.servers.grpc.server.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| defaults.server_config.servers.grpc.server.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| defaults.server_config.servers.grpc.server.socket_option.tcp_cork | bool | `false` | server listen socket option for tcp_cork functionality | +| defaults.server_config.servers.grpc.server.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| defaults.server_config.servers.grpc.server.socket_option.tcp_fast_open | bool | `false` | server listen socket option for tcp_fast_open functionality | +| defaults.server_config.servers.grpc.server.socket_option.tcp_no_delay | bool | `false` | server listen socket option for tcp_no_delay functionality | +| defaults.server_config.servers.grpc.server.socket_option.tcp_quick_ack | bool | `false` | server listen socket option for tcp_quick_ack functionality | +| defaults.server_config.servers.grpc.server.socket_path | string | `""` | mysql socket_path | +| defaults.server_config.servers.grpc.servicePort | int | `8081` | gRPC server service port | +| defaults.server_config.servers.rest.enabled | bool | `false` | REST server enabled | +| defaults.server_config.servers.rest.host | string | `"0.0.0.0"` | REST server host | +| defaults.server_config.servers.rest.port | int | `8080` | REST server port | +| defaults.server_config.servers.rest.server.http.handler_timeout | string | `"5s"` | REST server handler timeout | +| defaults.server_config.servers.rest.server.http.idle_timeout | string | `"2s"` | REST server idle timeout | +| defaults.server_config.servers.rest.server.http.read_header_timeout | string | `"1s"` | REST server read header timeout | +| defaults.server_config.servers.rest.server.http.read_timeout | string | `"1s"` | REST server read timeout | +| defaults.server_config.servers.rest.server.http.shutdown_duration | string | `"5s"` | REST server shutdown duration | +| defaults.server_config.servers.rest.server.http.write_timeout | string | `"1s"` | REST server write timeout | +| defaults.server_config.servers.rest.server.mode | string | `"REST"` | REST server server mode | +| defaults.server_config.servers.rest.server.network | string | `"tcp"` | mysql network | +| defaults.server_config.servers.rest.server.probe_wait_time | string | `"3s"` | REST server probe wait time | +| defaults.server_config.servers.rest.server.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| defaults.server_config.servers.rest.server.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| defaults.server_config.servers.rest.server.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| defaults.server_config.servers.rest.server.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| defaults.server_config.servers.rest.server.socket_option.tcp_cork | bool | `false` | server listen socket option for tcp_cork functionality | +| defaults.server_config.servers.rest.server.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| defaults.server_config.servers.rest.server.socket_option.tcp_fast_open | bool | `false` | server listen socket option for tcp_fast_open functionality | +| defaults.server_config.servers.rest.server.socket_option.tcp_no_delay | bool | `false` | server listen socket option for tcp_no_delay functionality | +| defaults.server_config.servers.rest.server.socket_option.tcp_quick_ack | bool | `false` | server listen socket option for tcp_quick_ack functionality | +| defaults.server_config.servers.rest.server.socket_path | string | `""` | mysql socket_path | +| defaults.server_config.servers.rest.servicePort | int | `8080` | REST server service port | +| defaults.server_config.tls.ca | string | `"/path/to/ca"` | TLS ca path | +| defaults.server_config.tls.cert | string | `"/path/to/cert"` | TLS cert path | +| defaults.server_config.tls.enabled | bool | `false` | TLS enabled | +| defaults.server_config.tls.insecure_skip_verify | bool | `false` | enable/disable skip SSL certificate verification | +| defaults.server_config.tls.key | string | `"/path/to/key"` | TLS key path | +| defaults.time_zone | string | `"UTC"` | Time zone | +| discoverer.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | +| discoverer.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | +| discoverer.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | +| discoverer.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity required scheduling terms | +| discoverer.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["vald-discoverer"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | pod anti-affinity preferred scheduling terms | +| discoverer.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | +| discoverer.annotations | object | `{}` | deployment annotations | +| discoverer.clusterRole.enabled | bool | `true` | creates clusterRole resource | +| discoverer.clusterRole.name | string | `"discoverer"` | name of clusterRole | +| discoverer.clusterRoleBinding.enabled | bool | `true` | creates clusterRoleBinding resource | +| discoverer.clusterRoleBinding.name | string | `"discoverer"` | name of clusterRoleBinding | +| discoverer.discoverer.discovery_duration | string | `"3s"` | duration to discovery | +| discoverer.discoverer.name | string | `""` | name to discovery | +| discoverer.discoverer.namespace | string | `"_MY_POD_NAMESPACE_"` | namespace to discovery | +| discoverer.discoverer.net.dialer.dual_stack_enabled | bool | `false` | TCP dialer dual stack enabled | +| discoverer.discoverer.net.dialer.keepalive | string | `"10m"` | TCP dialer keep alive | +| discoverer.discoverer.net.dialer.timeout | string | `"30s"` | TCP dialer timeout | +| discoverer.discoverer.net.dns.cache_enabled | bool | `true` | TCP DNS cache enabled | +| discoverer.discoverer.net.dns.cache_expiration | string | `"24h"` | TCP DNS cache expiration | +| discoverer.discoverer.net.dns.refresh_duration | string | `"5m"` | TCP DNS cache refresh duration | +| discoverer.discoverer.net.socket_option.ip_recover_destination_addr | bool | `false` | server listen socket option for ip_recover_destination_addr functionality | +| discoverer.discoverer.net.socket_option.ip_transparent | bool | `false` | server listen socket option for ip_transparent functionality | +| discoverer.discoverer.net.socket_option.reuse_addr | bool | `true` | server listen socket option for reuse_addr functionality | +| discoverer.discoverer.net.socket_option.reuse_port | bool | `true` | server listen socket option for reuse_port functionality | +| discoverer.discoverer.net.socket_option.tcp_cork | bool | `false` | server listen socket option for tcp_cork functionality | +| discoverer.discoverer.net.socket_option.tcp_defer_accept | bool | `false` | server listen socket option for tcp_defer_accept functionality | +| discoverer.discoverer.net.socket_option.tcp_fast_open | bool | `false` | server listen socket option for tcp_fast_open functionality | +| discoverer.discoverer.net.socket_option.tcp_no_delay | bool | `false` | server listen socket option for tcp_no_delay functionality | +| discoverer.discoverer.net.socket_option.tcp_quick_ack | bool | `false` | server listen socket option for tcp_quick_ack functionality | +| discoverer.discoverer.net.tls.ca | string | `"/path/to/ca"` | TLS ca path | +| discoverer.discoverer.net.tls.cert | string | `"/path/to/cert"` | TLS cert path | +| discoverer.discoverer.net.tls.enabled | bool | `false` | TLS enabled | +| discoverer.discoverer.net.tls.insecure_skip_verify | bool | `false` | enable/disable skip SSL certificate verification | +| discoverer.discoverer.net.tls.key | string | `"/path/to/key"` | TLS key path | +| discoverer.discoverer.selectors | object | `{"node":{"fields":{},"labels":{}},"node_metrics":{"fields":{},"labels":{}},"pod":{"fields":{},"labels":{}},"pod_metrics":{"fields":{},"labels":{}}}` | k8s resource selectors | +| discoverer.discoverer.selectors.node | object | `{"fields":{},"labels":{}}` | k8s resource selectors for node discovery | +| discoverer.discoverer.selectors.node.fields | object | `{}` | k8s field selectors for node discovery | +| discoverer.discoverer.selectors.node.labels | object | `{}` | k8s label selectors for node discovery | +| discoverer.discoverer.selectors.node_metrics | object | `{"fields":{},"labels":{}}` | k8s resource selectors for node_metrics discovery | +| discoverer.discoverer.selectors.node_metrics.fields | object | `{}` | k8s field selectors for node_metrics discovery | +| discoverer.discoverer.selectors.node_metrics.labels | object | `{}` | k8s label selectors for node_metrics discovery | +| discoverer.discoverer.selectors.pod | object | `{"fields":{},"labels":{}}` | k8s resource selectors for pod discovery | +| discoverer.discoverer.selectors.pod.fields | object | `{}` | k8s field selectors for pod discovery | +| discoverer.discoverer.selectors.pod.labels | object | `{}` | k8s label selectors for pod discovery | +| discoverer.discoverer.selectors.pod_metrics | object | `{"fields":{},"labels":{}}` | k8s resource selectors for pod_metrics discovery | +| discoverer.discoverer.selectors.pod_metrics.fields | object | `{}` | k8s field selectors for pod_metrics discovery | +| discoverer.discoverer.selectors.pod_metrics.labels | object | `{}` | k8s label selectors for pod_metrics discovery | +| discoverer.enabled | bool | `true` | discoverer enabled | +| discoverer.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | +| discoverer.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| discoverer.hpa.enabled | bool | `false` | HPA enabled | +| discoverer.hpa.targetCPUUtilizationPercentage | int | `80` | HPA CPU utilization percentage | +| discoverer.image.pullPolicy | string | `"Always"` | image pull policy | +| discoverer.image.repository | string | `"vdaas/vald-discoverer-k8s"` | image repository | +| discoverer.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| discoverer.initContainers | list | `[]` | init containers | +| discoverer.internalTrafficPolicy | string | `""` | internal traffic policy : Cluster or Local | +| discoverer.kind | string | `"Deployment"` | deployment kind: Deployment or DaemonSet | +| discoverer.logging | object | `{}` | logging config (overrides defaults.logging) | +| discoverer.maxReplicas | int | `2` | maximum number of replicas. if HPA is disabled, this value will be ignored. | +| discoverer.maxUnavailable | string | `"50%"` | maximum number of unavailable replicas | +| discoverer.minReplicas | int | `1` | minimum number of replicas. if HPA is disabled, the replicas will be set to this value | +| discoverer.name | string | `"vald-discoverer"` | name of discoverer deployment | +| discoverer.nodeName | string | `""` | node name | +| discoverer.nodeSelector | object | `{}` | node selector | +| discoverer.observability | object | `{"otlp":{"attribute":{"service_name":"vald-discoverer"}}}` | observability config (overrides defaults.observability) | +| discoverer.podAnnotations | object | `{}` | pod annotations | +| discoverer.podPriority.enabled | bool | `true` | discoverer pod PriorityClass enabled | +| discoverer.podPriority.value | int | `1000000` | discoverer pod PriorityClass value | +| discoverer.podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | +| discoverer.progressDeadlineSeconds | int | `600` | progress deadline seconds | +| discoverer.resources | object | `{"limits":{"cpu":"600m","memory":"200Mi"},"requests":{"cpu":"200m","memory":"65Mi"}}` | compute resources | +| discoverer.revisionHistoryLimit | int | `2` | number of old history to retain to allow rollback | +| discoverer.rollingUpdate.maxSurge | string | `"25%"` | max surge of rolling update | +| discoverer.rollingUpdate.maxUnavailable | string | `"25%"` | max unavailable of rolling update | +| discoverer.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for container | +| discoverer.server_config | object | `{"healths":{"liveness":{},"readiness":{},"startup":{}},"metrics":{"pprof":{}},"servers":{"grpc":{},"rest":{}}}` | server config (overrides defaults.server_config) | +| discoverer.service.annotations | object | `{}` | service annotations | +| discoverer.service.labels | object | `{}` | service labels | +| discoverer.serviceAccount.enabled | bool | `true` | creates service account | +| discoverer.serviceAccount.name | string | `"vald"` | name of service account | +| discoverer.serviceType | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| discoverer.terminationGracePeriodSeconds | int | `30` | duration in seconds pod needs to terminate gracefully | +| discoverer.time_zone | string | `""` | Time zone | +| discoverer.tolerations | list | `[]` | tolerations | +| discoverer.topologySpreadConstraints | list | `[]` | topology spread constraints of discoverer pods | +| discoverer.version | string | `"v0.0.0"` | version of discoverer config | +| discoverer.volumeMounts | list | `[]` | volume mounts | +| discoverer.volumes | list | `[]` | volumes | +| gateway.filter.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | +| gateway.filter.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | +| gateway.filter.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | +| gateway.filter.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity required scheduling terms | +| gateway.filter.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["vald-filter-gateway"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | pod anti-affinity preferred scheduling terms | +| gateway.filter.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | +| gateway.filter.annotations | object | `{}` | deployment annotations | +| gateway.filter.enabled | bool | `false` | gateway enabled | +| gateway.filter.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | +| gateway.filter.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| gateway.filter.gateway_config.egress_filter | object | `{"client":{},"distance_filters":[],"object_filters":[]}` | gRPC client config for egress filter | +| gateway.filter.gateway_config.egress_filter.client | object | `{}` | gRPC client config for egress filter (overrides defaults.grpc.client) | +| gateway.filter.gateway_config.egress_filter.distance_filters | list | `[]` | distance egress vector filter targets | +| gateway.filter.gateway_config.egress_filter.object_filters | list | `[]` | object egress vector filter targets | +| gateway.filter.gateway_config.gateway_client | object | `{}` | gRPC client for next gateway (overrides defaults.grpc.client) | +| gateway.filter.gateway_config.ingress_filter | object | `{"client":{},"insert_filters":[],"search_filters":[],"update_filters":[],"upsert_filters":[],"vectorizer":""}` | gRPC client config for ingress filter | +| gateway.filter.gateway_config.ingress_filter.client | object | `{}` | gRPC client for ingress filter (overrides defaults.grpc.client) | +| gateway.filter.gateway_config.ingress_filter.insert_filters | list | `[]` | insert ingress vector filter targets | +| gateway.filter.gateway_config.ingress_filter.search_filters | list | `[]` | search ingress vector filter targets | +| gateway.filter.gateway_config.ingress_filter.update_filters | list | `[]` | update ingress vector filter targets | +| gateway.filter.gateway_config.ingress_filter.upsert_filters | list | `[]` | upsert ingress vector filter targets | +| gateway.filter.gateway_config.ingress_filter.vectorizer | string | `""` | object ingress vectorize filter targets | +| gateway.filter.hpa.enabled | bool | `true` | HPA enabled | +| gateway.filter.hpa.targetCPUUtilizationPercentage | int | `80` | HPA CPU utilization percentage | +| gateway.filter.image.pullPolicy | string | `"Always"` | image pull policy | +| gateway.filter.image.repository | string | `"vdaas/vald-filter-gateway"` | image repository | +| gateway.filter.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| gateway.filter.ingress.annotations | object | `{"nginx.ingress.kubernetes.io/grpc-backend":"true"}` | annotations for ingress | +| gateway.filter.ingress.defaultBackend | object | `{"enabled":true}` | defaultBackend config | +| gateway.filter.ingress.defaultBackend.enabled | bool | `true` | gateway ingress defaultBackend enabled | +| gateway.filter.ingress.enabled | bool | `false` | gateway ingress enabled | +| gateway.filter.ingress.host | string | `"filter.gateway.vald.vdaas.org"` | ingress hostname | +| gateway.filter.ingress.pathType | string | `"ImplementationSpecific"` | gateway ingress pathType | +| gateway.filter.ingress.servicePort | string | `"grpc"` | service port to be exposed by ingress | +| gateway.filter.initContainers | list | `[{"image":"busybox:stable","name":"wait-for-gateway-lb","sleepDuration":2,"target":"gateway-lb","type":"wait-for"}]` | init containers | +| gateway.filter.internalTrafficPolicy | string | `""` | internal traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| gateway.filter.kind | string | `"Deployment"` | deployment kind: Deployment or DaemonSet | +| gateway.filter.logging | object | `{}` | logging config (overrides defaults.logging) | +| gateway.filter.maxReplicas | int | `9` | maximum number of replicas. if HPA is disabled, this value will be ignored. | +| gateway.filter.maxUnavailable | string | `"50%"` | maximum number of unavailable replicas | +| gateway.filter.minReplicas | int | `3` | minimum number of replicas. if HPA is disabled, the replicas will be set to this value | +| gateway.filter.name | string | `"vald-filter-gateway"` | name of filter gateway deployment | +| gateway.filter.nodeName | string | `""` | node name | +| gateway.filter.nodeSelector | object | `{}` | node selector | +| gateway.filter.observability | object | `{"otlp":{"attribute":{"service_name":"vald-filter-gateway"}}}` | observability config (overrides defaults.observability) | +| gateway.filter.podAnnotations | object | `{}` | pod annotations | +| gateway.filter.podPriority.enabled | bool | `true` | gateway pod PriorityClass enabled | +| gateway.filter.podPriority.value | int | `1000000` | gateway pod PriorityClass value | +| gateway.filter.podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | +| gateway.filter.progressDeadlineSeconds | int | `600` | progress deadline seconds | +| gateway.filter.resources | object | `{"limits":{"cpu":"2000m","memory":"700Mi"},"requests":{"cpu":"200m","memory":"150Mi"}}` | compute resources | +| gateway.filter.revisionHistoryLimit | int | `2` | number of old history to retain to allow rollback | +| gateway.filter.rollingUpdate.maxSurge | string | `"25%"` | max surge of rolling update | +| gateway.filter.rollingUpdate.maxUnavailable | string | `"25%"` | max unavailable of rolling update | +| gateway.filter.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for container | +| gateway.filter.server_config | object | `{"healths":{"liveness":{},"readiness":{},"startup":{}},"metrics":{"pprof":{}},"servers":{"grpc":{},"rest":{}}}` | server config (overrides defaults.server_config) | +| gateway.filter.service.annotations | object | `{}` | service annotations | +| gateway.filter.service.labels | object | `{}` | service labels | +| gateway.filter.serviceType | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| gateway.filter.terminationGracePeriodSeconds | int | `30` | duration in seconds pod needs to terminate gracefully | +| gateway.filter.time_zone | string | `""` | Time zone | +| gateway.filter.tolerations | list | `[]` | tolerations | +| gateway.filter.topologySpreadConstraints | list | `[]` | topology spread constraints of gateway pods | +| gateway.filter.version | string | `"v0.0.0"` | version of gateway config | +| gateway.filter.volumeMounts | list | `[]` | volume mounts | +| gateway.filter.volumes | list | `[]` | volumes | +| gateway.lb.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | +| gateway.lb.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | +| gateway.lb.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | +| gateway.lb.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity required scheduling terms | +| gateway.lb.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["vald-lb-gateway"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | pod anti-affinity preferred scheduling terms | +| gateway.lb.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | +| gateway.lb.annotations | object | `{}` | deployment annotations | +| gateway.lb.enabled | bool | `true` | gateway enabled | +| gateway.lb.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | +| gateway.lb.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| gateway.lb.gateway_config.agent_namespace | string | `"_MY_POD_NAMESPACE_"` | agent namespace | +| gateway.lb.gateway_config.discoverer.agent_client_options | object | `{}` | gRPC client options for agents (overrides defaults.grpc.client) | +| gateway.lb.gateway_config.discoverer.client | object | `{}` | gRPC client for discoverer (overrides defaults.grpc.client) | +| gateway.lb.gateway_config.discoverer.duration | string | `"200ms"` | | +| gateway.lb.gateway_config.index_replica | int | `3` | number of index replica | +| gateway.lb.gateway_config.multi_operation_concurrency | int | `20` | number of concurrency of multiXXX api's operation | +| gateway.lb.gateway_config.node_name | string | `""` | node name | +| gateway.lb.hpa.enabled | bool | `true` | HPA enabled | +| gateway.lb.hpa.targetCPUUtilizationPercentage | int | `80` | HPA CPU utilization percentage | +| gateway.lb.image.pullPolicy | string | `"Always"` | image pull policy | +| gateway.lb.image.repository | string | `"vdaas/vald-lb-gateway"` | image repository | +| gateway.lb.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| gateway.lb.ingress.annotations | object | `{"nginx.ingress.kubernetes.io/grpc-backend":"true"}` | annotations for ingress | +| gateway.lb.ingress.defaultBackend | object | `{"enabled":true}` | defaultBackend config | +| gateway.lb.ingress.defaultBackend.enabled | bool | `true` | gateway ingress defaultBackend enabled | +| gateway.lb.ingress.enabled | bool | `false` | gateway ingress enabled | +| gateway.lb.ingress.host | string | `"lb.gateway.vald.vdaas.org"` | ingress hostname | +| gateway.lb.ingress.pathType | string | `"ImplementationSpecific"` | gateway ingress pathType | +| gateway.lb.ingress.servicePort | string | `"grpc"` | service port to be exposed by ingress | +| gateway.lb.initContainers | list | `[{"image":"busybox:stable","name":"wait-for-discoverer","sleepDuration":2,"target":"discoverer","type":"wait-for"},{"image":"busybox:stable","name":"wait-for-agent","sleepDuration":2,"target":"agent","type":"wait-for"}]` | init containers | +| gateway.lb.internalTrafficPolicy | string | `""` | internal traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| gateway.lb.kind | string | `"Deployment"` | deployment kind: Deployment or DaemonSet | +| gateway.lb.logging | object | `{}` | logging config (overrides defaults.logging) | +| gateway.lb.maxReplicas | int | `9` | maximum number of replicas. if HPA is disabled, this value will be ignored. | +| gateway.lb.maxUnavailable | string | `"50%"` | maximum number of unavailable replicas | +| gateway.lb.minReplicas | int | `3` | minimum number of replicas. if HPA is disabled, the replicas will be set to this value | +| gateway.lb.name | string | `"vald-lb-gateway"` | name of gateway deployment | +| gateway.lb.nodeName | string | `""` | node name | +| gateway.lb.nodeSelector | object | `{}` | node selector | +| gateway.lb.observability | object | `{"otlp":{"attribute":{"service_name":"vald-lb-gateway"}}}` | observability config (overrides defaults.observability) | +| gateway.lb.podAnnotations | object | `{}` | pod annotations | +| gateway.lb.podPriority.enabled | bool | `true` | gateway pod PriorityClass enabled | +| gateway.lb.podPriority.value | int | `1000000` | gateway pod PriorityClass value | +| gateway.lb.podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | +| gateway.lb.progressDeadlineSeconds | int | `600` | progress deadline seconds | +| gateway.lb.resources | object | `{"limits":{"cpu":"2000m","memory":"700Mi"},"requests":{"cpu":"200m","memory":"150Mi"}}` | compute resources | +| gateway.lb.revisionHistoryLimit | int | `2` | number of old history to retain to allow rollback | +| gateway.lb.rollingUpdate.maxSurge | string | `"25%"` | max surge of rolling update | +| gateway.lb.rollingUpdate.maxUnavailable | string | `"25%"` | max unavailable of rolling update | +| gateway.lb.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for container | +| gateway.lb.server_config | object | `{"healths":{"liveness":{},"readiness":{},"startup":{}},"metrics":{"pprof":{}},"servers":{"grpc":{},"rest":{}}}` | server config (overrides defaults.server_config) | +| gateway.lb.service.annotations | object | `{}` | service annotations | +| gateway.lb.service.labels | object | `{}` | service labels | +| gateway.lb.serviceType | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| gateway.lb.terminationGracePeriodSeconds | int | `30` | duration in seconds pod needs to terminate gracefully | +| gateway.lb.time_zone | string | `""` | Time zone | +| gateway.lb.tolerations | list | `[]` | tolerations | +| gateway.lb.topologySpreadConstraints | list | `[]` | topology spread constraints of gateway pods | +| gateway.lb.version | string | `"v0.0.0"` | version of gateway config | +| gateway.lb.volumeMounts | list | `[]` | volume mounts | +| gateway.lb.volumes | list | `[]` | volumes | +| manager.index.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | node affinity preferred scheduling terms | +| manager.index.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | +| manager.index.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | +| manager.index.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity required scheduling terms | +| manager.index.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity preferred scheduling terms | +| manager.index.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | +| manager.index.annotations | object | `{}` | deployment annotations | +| manager.index.enabled | bool | `true` | index manager enabled | +| manager.index.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | +| manager.index.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | +| manager.index.image.pullPolicy | string | `"Always"` | image pull policy | +| manager.index.image.repository | string | `"vdaas/vald-manager-index"` | image repository | +| manager.index.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| manager.index.indexer.agent_namespace | string | `"_MY_POD_NAMESPACE_"` | namespace of agent pods to manage | +| manager.index.indexer.auto_index_check_duration | string | `"1m"` | check duration of automatic indexing | +| manager.index.indexer.auto_index_duration_limit | string | `"30m"` | limit duration of automatic indexing | +| manager.index.indexer.auto_index_length | int | `100` | number of cache to trigger automatic indexing | +| manager.index.indexer.auto_save_index_duration_limit | string | `"3h"` | limit duration of automatic index saving | +| manager.index.indexer.auto_save_index_wait_duration | string | `"10m"` | duration of automatic index saving wait duration for next saving | +| manager.index.indexer.concurrency | int | `1` | concurrency | +| manager.index.indexer.creation_pool_size | int | `10000` | number of pool size of create index processing | +| manager.index.indexer.discoverer.agent_client_options | object | `{"dial_option":{"net":{"dialer":{"keepalive":"15m"}}}}` | gRPC client options for agents (overrides defaults.grpc.client) | +| manager.index.indexer.discoverer.client | object | `{}` | gRPC client for discoverer (overrides defaults.grpc.client) | +| manager.index.indexer.discoverer.duration | string | `"500ms"` | refresh duration to discover | +| manager.index.indexer.node_name | string | `""` | node name | +| manager.index.initContainers | list | `[{"image":"busybox:stable","name":"wait-for-agent","sleepDuration":2,"target":"agent","type":"wait-for"},{"image":"busybox:stable","name":"wait-for-discoverer","sleepDuration":2,"target":"discoverer","type":"wait-for"}]` | init containers | +| manager.index.kind | string | `"Deployment"` | deployment kind: Deployment or DaemonSet | +| manager.index.logging | object | `{}` | logging config (overrides defaults.logging) | +| manager.index.maxUnavailable | string | `"50%"` | maximum number of unavailable replicas | +| manager.index.name | string | `"vald-manager-index"` | name of index manager deployment | +| manager.index.nodeName | string | `""` | node name | +| manager.index.nodeSelector | object | `{}` | node selector | +| manager.index.observability | object | `{"otlp":{"attribute":{"service_name":"vald-manager-index"}}}` | observability config (overrides defaults.observability) | +| manager.index.podAnnotations | object | `{}` | pod annotations | +| manager.index.podPriority.enabled | bool | `true` | index manager pod PriorityClass enabled | +| manager.index.podPriority.value | int | `1000000` | index manager pod PriorityClass value | +| manager.index.podSecurityContext | object | `{"fsGroup":65532,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for pod | +| manager.index.progressDeadlineSeconds | int | `600` | progress deadline seconds | +| manager.index.replicas | int | `1` | number of replicas | +| manager.index.resources | object | `{"limits":{"cpu":"1000m","memory":"500Mi"},"requests":{"cpu":"200m","memory":"80Mi"}}` | compute resources | +| manager.index.revisionHistoryLimit | int | `2` | number of old history to retain to allow rollback | +| manager.index.rollingUpdate.maxSurge | string | `"25%"` | max surge of rolling update | +| manager.index.rollingUpdate.maxUnavailable | string | `"25%"` | max unavailable of rolling update | +| manager.index.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532}` | security context for container | +| manager.index.server_config | object | `{"healths":{"liveness":{},"readiness":{},"startup":{}},"metrics":{"pprof":{}},"servers":{"grpc":{},"rest":{}}}` | server config (overrides defaults.server_config) | +| manager.index.service.annotations | object | `{}` | service annotations | +| manager.index.service.labels | object | `{}` | service labels | +| manager.index.serviceType | string | `"ClusterIP"` | service type: ClusterIP, LoadBalancer or NodePort | +| manager.index.terminationGracePeriodSeconds | int | `30` | duration in seconds pod needs to terminate gracefully | +| manager.index.time_zone | string | `""` | Time zone | +| manager.index.tolerations | list | `[]` | tolerations | +| manager.index.topologySpreadConstraints | list | `[]` | topology spread constraints of index manager pods | +| manager.index.version | string | `"v0.0.0"` | version of index manager config | +| manager.index.volumeMounts | list | `[]` | volume mounts | +| manager.index.volumes | list | `[]` | volumes | +>>>>>>> feature/agent/new-agents-helm-template-base diff --git a/charts/vald/templates/agent/faiss/configmap.yaml b/charts/vald/templates/agent/faiss/configmap.yaml new file mode 100644 index 00000000000..53ff21f77d7 --- /dev/null +++ b/charts/vald/templates/agent/faiss/configmap.yaml @@ -0,0 +1,45 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $agent := .Values.agent -}} +{{- if and ($agent.enabled) (eq (lower $agent.algorithm) "faiss")}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $agent.name }}-config + labels: + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} + app.kubernetes.io/component: agent +data: + config.yaml: | + --- + version: {{ $agent.version }} + time_zone: {{ default .Values.defaults.time_zone $agent.time_zone }} + logging: + {{- $logging := dict "Values" $agent.logging "default" .Values.defaults.logging }} + {{- include "vald.logging" $logging | nindent 6 }} + server_config: + {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config }} + {{- include "vald.servers" $servers | nindent 6 }} + observability: + {{- $observability := dict "Values" $agent.observability "default" .Values.defaults.observability }} + {{- include "vald.observability" $observability | nindent 6 }} + faiss: + {{- toYaml $agent.faiss | nindent 6 }} +{{- end }} diff --git a/charts/vald/templates/agent/configmap.yaml b/charts/vald/templates/agent/ngt/configmap.yaml similarity index 96% rename from charts/vald/templates/agent/configmap.yaml rename to charts/vald/templates/agent/ngt/configmap.yaml index 0d6785ca4a0..cf14a1686bc 100644 --- a/charts/vald/templates/agent/configmap.yaml +++ b/charts/vald/templates/agent/ngt/configmap.yaml @@ -14,7 +14,7 @@ # limitations under the License. # {{- $agent := .Values.agent -}} -{{- if $agent.enabled }} +{{- if and ($agent.enabled) (eq (lower $agent.algorithm) "ngt")}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/vald/templates/agent/qbg/configmap.yaml b/charts/vald/templates/agent/qbg/configmap.yaml new file mode 100644 index 00000000000..d1350f7ccc3 --- /dev/null +++ b/charts/vald/templates/agent/qbg/configmap.yaml @@ -0,0 +1,45 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $agent := .Values.agent -}} +{{- if and ($agent.enabled) (eq (lower $agent.algorithm) "qbg")}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $agent.name }}-config + labels: + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} + app.kubernetes.io/component: agent +data: + config.yaml: | + --- + version: {{ $agent.version }} + time_zone: {{ default .Values.defaults.time_zone $agent.time_zone }} + logging: + {{- $logging := dict "Values" $agent.logging "default" .Values.defaults.logging }} + {{- include "vald.logging" $logging | nindent 6 }} + server_config: + {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config }} + {{- include "vald.servers" $servers | nindent 6 }} + observability: + {{- $observability := dict "Values" $agent.observability "default" .Values.defaults.observability }} + {{- include "vald.observability" $observability | nindent 6 }} + qbg: + {{- toYaml $agent.qbg | nindent 6 }} +{{- end }} diff --git a/charts/vald/templates/agent/sidecar-configmap.yaml b/charts/vald/templates/agent/sidecar/configmap.yaml similarity index 100% rename from charts/vald/templates/agent/sidecar-configmap.yaml rename to charts/vald/templates/agent/sidecar/configmap.yaml diff --git a/charts/vald/templates/agent/sidecar-svc.yaml b/charts/vald/templates/agent/sidecar/svc.yaml similarity index 100% rename from charts/vald/templates/agent/sidecar-svc.yaml rename to charts/vald/templates/agent/sidecar/svc.yaml diff --git a/charts/vald/templates/agent/statefulset.yaml b/charts/vald/templates/agent/statefulset.yaml index 2051680a7ea..4dc1af3a455 100644 --- a/charts/vald/templates/agent/statefulset.yaml +++ b/charts/vald/templates/agent/statefulset.yaml @@ -101,6 +101,7 @@ spec: volumeMounts: - name: {{ $agent.sidecar.name }}-config mountPath: /etc/server/ + {{- if eq $agent.algorithm "ngt" }} {{- if not $agent.ngt.enable_in_memory_mode }} {{- if $agent.ngt.index_path }} {{- if $agent.persistentVolume.enabled }} @@ -113,6 +114,7 @@ spec: {{- end }} {{- end }} {{- end }} + {{- end }} {{- if $agent.volumeMounts }} {{- toYaml $agent.volumeMounts | nindent 12 }} {{- end }} diff --git a/charts/vald/values.schema.json b/charts/vald/values.schema.json index f225cea11d0..03dc8c7fe3e 100644 --- a/charts/vald/values.schema.json +++ b/charts/vald/values.schema.json @@ -61,6 +61,11 @@ } } }, + "algorithm": { + "type": "string", + "description": "agent algorithm type. it should be `ngt` or `qbg` or `faiss`.", + "enum": ["ngt", "qbg", "faiss"] + }, "annotations": { "type": "object", "description": "deployment annotations" @@ -341,7 +346,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -1114,6 +1119,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -1950,7 +1959,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -2604,6 +2613,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -3394,7 +3407,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -4034,6 +4047,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -4753,7 +4770,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -5448,6 +5465,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -7044,7 +7065,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -7745,6 +7766,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -8992,7 +9017,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -9693,6 +9718,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -14362,7 +14391,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -17905,6 +17934,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index f97f20b34d2..82c9073f4d4 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -210,6 +210,9 @@ defaults: # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.enable_reflection", "type": "boolean"} # defaults.server_config.servers.grpc.server.grpc.enable_reflection -- gRPC server reflection option enable_reflection: true + # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.enable_admin", "type": "boolean"} + # defaults.server_config.servers.grpc.server.grpc.enable_admin -- gRPC server admin option + enable_admin: true # @schema {"name": "defaults.server_config.servers.grpc.server.socket_option", "alias": "socket_option"} socket_option: # defaults.server_config.servers.grpc.server.socket_option.reuse_port -- server listen socket option for reuse_port functionality @@ -805,7 +808,7 @@ defaults: # @schema {"name": "defaults.observability.metrics.enable_version_info", "type": "boolean"} # defaults.observability.metrics.enable_version_info -- version info metrics enabled enable_version_info: true - # @schema {"name": "defaults.observability.metrics.version_info_labels", "type": "array", "items": {"type": "string", "enum": ["vald_version", "server_name", "git_commit", "build_time", "go_version", "go_os", "go_arch", "cgo_enabled", "ngt_version", "build_cpu_info_flags"]}} + # @schema {"name": "defaults.observability.metrics.version_info_labels", "type": "array", "items": {"type": "string", "enum": ["vald_version", "server_name", "git_commit", "build_time", "go_version", "go_os", "go_arch", "cgo_enabled", "algorithm_info", "build_cpu_info_flags"]}} # defaults.observability.metrics.version_info_labels -- enabled label names of version info version_info_labels: - "vald_version" @@ -815,7 +818,7 @@ defaults: - "go_version" - "go_os" - "go_arch" - - "ngt_version" + - "algorithm_info" # @schema {"name": "defaults.observability.metrics.enable_memory", "type": "boolean"} # defaults.observability.metrics.enable_memory -- memory metrics enabled enable_memory: true @@ -1420,6 +1423,10 @@ agent: # @schema {"name": "agent.version", "alias": "version"} # agent.version -- version of agent config version: v0.0.0 + # @schema {"name": "agent.algorithm", "type": "string", "enum": ["ngt", "qbg", "faiss"]} + # agent.algorithm -- agent algorithm type. + # it should be `ngt` or `qbg` or `faiss`. + algorithm: ngt # @schema {"name": "agent.time_zone", "type": "string"} # agent.time_zone -- Time zone time_zone: "" @@ -1428,7 +1435,7 @@ agent: logging: {} # @schema {"name": "agent.name", "type": "string"} # agent.name -- name of agent deployment - name: vald-agent-ngt + name: vald-agent # @schema {"name": "agent.kind", "type": "string", "enum": ["StatefulSet", "Deployment", "DaemonSet"]} # agent.kind -- deployment kind: Deployment, DaemonSet or StatefulSet kind: StatefulSet @@ -1599,7 +1606,7 @@ agent: - key: app operator: In values: - - vald-agent-ngt + - vald-agent # agent.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] # @schema {"name": "agent.topologySpreadConstraints", "alias": "topologySpreadConstraints"} @@ -1625,7 +1632,7 @@ agent: observability: otlp: attribute: - service_name: vald-agent-ngt + service_name: vald-agent # @schema {"name": "agent.resources", "alias": "resources"} # agent.resources -- compute resources. # recommended setting of memory requests = cluster memory * 0.4 / number of agent pods diff --git a/charts/vald/values/dev.yaml b/charts/vald/values/dev.yaml index 6bbd9f6cf09..17646ac379d 100644 --- a/charts/vald/values/dev.yaml +++ b/charts/vald/values/dev.yaml @@ -28,6 +28,8 @@ gateway: profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: "vald-lb-gateway" + ingress: + enabled: true resources: requests: cpu: 100m diff --git a/cmd/agent/core/faiss/main.go b/cmd/agent/core/faiss/main.go new file mode 100644 index 00000000000..d2aba32de10 --- /dev/null +++ b/cmd/agent/core/faiss/main.go @@ -0,0 +1,59 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package main provides program main +package main + +import ( + "context" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/pkg/agent/core/faiss/config" + "github.com/vdaas/vald/pkg/agent/core/faiss/usecase" +) + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "agent faiss" +) + +func main() { + if err := safety.RecoverFunc(func() error { + return runner.Do( + context.Background(), + runner.WithName(name), + runner.WithVersion(info.Version, maxVersion, minVersion), + runner.WithConfigLoader(func(path string) (interface{}, *config.GlobalConfig, error) { + cfg, err := config.NewConfig(path) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") + } + return cfg, &cfg.GlobalConfig, nil + }), + runner.WithDaemonInitializer(func(cfg interface{}) (runner.Runner, error) { + return usecase.New(cfg.(*config.Data)) + }), + ) + })(); err != nil { + log.Fatal(err, info.Get()) + return + } +} diff --git a/cmd/agent/core/faiss/main_test.go b/cmd/agent/core/faiss/main_test.go new file mode 100644 index 00000000000..6d5b51e2013 --- /dev/null +++ b/cmd/agent/core/faiss/main_test.go @@ -0,0 +1,87 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package faiss + +// NOT IMPLEMENTED BELOW +// +// func Test_main(t *testing.T) { +// type want struct { +// } +// type test struct { +// name string +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// main() +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } diff --git a/cmd/agent/core/faiss/sample.yaml b/cmd/agent/core/faiss/sample.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cmd/agent/core/ngt/sample-cow.yaml b/cmd/agent/core/ngt/sample-cow.yaml index 8a90e8ba377..90d62e02c5a 100644 --- a/cmd/agent/core/ngt/sample-cow.yaml +++ b/cmd/agent/core/ngt/sample-cow.yaml @@ -87,7 +87,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/cmd/agent/core/ngt/sample.yaml b/cmd/agent/core/ngt/sample.yaml index b0dbda74f1f..0b2b394b923 100644 --- a/cmd/agent/core/ngt/sample.yaml +++ b/cmd/agent/core/ngt/sample.yaml @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # - ---- version: v0.0.0 -time_zone: JST +time_zone: UTC logging: format: raw level: debug @@ -29,25 +27,66 @@ server_config: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true + enable_reflection: true header_table_size: 0 - initial_conn_window_size: 0 - initial_window_size: 0 - interceptors: [] + initial_conn_window_size: 2097152 + initial_window_size: 1048576 + interceptors: + - RecoverInterceptor keepalive: max_conn_age: "" max_conn_age_grace: "" max_conn_idle: "" - time: "" - timeout: "" + min_time: 10m + permit_without_stream: false + time: 3h + timeout: 60s max_header_list_size: 0 max_receive_message_size: 0 max_send_message_size: 0 read_buffer_size: 0 write_buffer_size: 0 mode: GRPC + network: tcp probe_wait_time: 3s restart: true + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" health_check_servers: + - name: liveness + host: 0.0.0.0 + port: 3000 + http: + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + shutdown_duration: 5s + write_timeout: "" + mode: "" + network: tcp + probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" - name: readiness host: 0.0.0.0 port: 3001 @@ -59,71 +98,118 @@ server_config: shutdown_duration: 0s write_timeout: "" mode: "" + network: tcp probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" metrics_servers: + - name: pprof + host: 0.0.0.0 + port: 6060 + http: + handler_timeout: 5s + idle_timeout: 2s + read_header_timeout: 1s + read_timeout: 1s + shutdown_duration: 5s + write_timeout: 1m + mode: REST + network: tcp + probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: true + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" startup_strategy: + - liveness + - pprof - grpc - readiness + shutdown_strategy: + - readiness + - grpc + - pprof + - liveness full_shutdown_duration: 600s tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key observability: enabled: false - collector: - duration: 5s - metrics: - enable_cgo: true - enable_goroutine: true - enable_memory: true - enable_version_info: true - version_info_labels: - - vald_version - - server_name - - git_commit - - build_time - - go_version - - go_os - - go_arch - - ngt_version + otlp: + collector_endpoint: "" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-agent" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - algorithm_info trace: enabled: false - sampling_rate: 1 - prometheus: - enabled: false - endpoint: /metrics - namespace: vald - jaeger: - enabled: false - collector_endpoint: "" - agent_endpoint: "jaeger-agent.default.svc.cluster.local:6831" - username: "" - password: "" - service_name: "vald-agent-ngt" - buffer_max_count: 10 ngt: auto_create_index_pool_size: 10000 auto_index_check_duration: 30m auto_index_duration_limit: 24h auto_index_length: 100 auto_save_index_duration: 35m + broken_index_history_limit: 0 bulk_insert_chunk_size: 10 creation_edge_size: 20 default_epsilon: 0.1 - default_pool_size: 100 + default_pool_size: 10000 default_radius: -1 - dimension: 100 + dimension: 784 distance_type: l2 + enable_copy_on_write: false enable_in_memory_mode: true - enable_proactive_gc: true + enable_proactive_gc: false index_path: "" initial_delay_max_duration: 3m + kvsdb: + concurrency: 6 load_index_timeout_factor: 1ms max_load_index_timeout: 10m min_load_index_timeout: 3m object_type: float search_edge_size: 10 - enable_copy_on_write: false - broken_index_history_limit: 0 + vqueue: + delete_buffer_pool_size: 5000 + insert_buffer_pool_size: 10000 diff --git a/cmd/discoverer/k8s/sample.yaml b/cmd/discoverer/k8s/sample.yaml index 564b1b877b2..92758d698af 100644 --- a/cmd/discoverer/k8s/sample.yaml +++ b/cmd/discoverer/k8s/sample.yaml @@ -14,38 +14,54 @@ # limitations under the License. # version: v0.0.0 -time_zone: JST +time_zone: UTC logging: format: raw - level: info + level: debug logger: glg server_config: servers: - name: grpc host: 0.0.0.0 - port: 8082 + port: 8081 grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true + enable_reflection: true header_table_size: 0 - initial_conn_window_size: 0 - initial_window_size: 0 + initial_conn_window_size: 2097152 + initial_window_size: 1048576 interceptors: - RecoverInterceptor keepalive: max_conn_age: "" max_conn_age_grace: "" max_conn_idle: "" - time: "" - timeout: "" + min_time: 10m + permit_without_stream: false + time: 3h + timeout: 60s max_header_list_size: 0 max_receive_message_size: 0 max_send_message_size: 0 read_buffer_size: 0 write_buffer_size: 0 mode: GRPC + network: tcp probe_wait_time: 3s restart: true + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" health_check_servers: - name: liveness host: 0.0.0.0 @@ -58,7 +74,19 @@ server_config: shutdown_duration: 5s write_timeout: "" mode: "" + network: tcp probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" - name: readiness host: 0.0.0.0 port: 3001 @@ -70,12 +98,54 @@ server_config: shutdown_duration: 0s write_timeout: "" mode: "" + network: tcp probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" metrics_servers: + - name: pprof + host: 0.0.0.0 + port: 6060 + http: + handler_timeout: 5s + idle_timeout: 2s + read_header_timeout: 1s + read_timeout: 1s + shutdown_duration: 5s + write_timeout: 1m + mode: REST + network: tcp + probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: true + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" startup_strategy: - liveness + - pprof - grpc - readiness + shutdown_strategy: + - readiness + - grpc + - pprof + - liveness full_shutdown_duration: 600s tls: ca: /path/to/ca @@ -85,42 +155,57 @@ server_config: key: /path/to/key observability: enabled: false - collector: - duration: 5s - metrics: - enable_cgo: true - enable_goroutine: true - enable_memory: true - enable_version_info: true - version_info_labels: - - vald_version - - server_name - - git_commit - - build_time - - go_version - - go_os - - go_arch - - ngt_version + otlp: + collector_endpoint: "" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-discoverer" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - algorithm_info trace: enabled: false - sampling_rate: 1 - prometheus: - enabled: false - endpoint: /metrics - namespace: vald - jaeger: - enabled: false - collector_endpoint: "" - agent_endpoint: "jaeger-agent.default.svc.cluster.local:6831" - username: "" - password: "" - service_name: "vald-discoverer" - buffer_max_count: 10 discoverer: - discovery_duration: 3s name: "" - namespace: _MY_POD_NAMESPACE_ - tcp: + namespace: "_MY_POD_NAMESPACE_" + discovery_duration: 3s + selectors: + pod: + labels: + app.kubernetes.io/component: agent + app.kubernetes.io/instance: release-name + fields: + status.phase: Running + node: + labels: {} + fields: {} + pod_metrics: + labels: {} + fields: + containers.name: vald-agent + node_metrics: + labels: {} + fields: {} + net: dialer: dual_stack_enabled: false keepalive: 10m @@ -129,9 +214,19 @@ discoverer: cache_enabled: true cache_expiration: 24h refresh_duration: 5m + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false tls: - enabled: false ca: /path/to/ca cert: /path/to/cert - key: /path/to/key + enabled: false insecure_skip_verify: false + key: /path/to/key diff --git a/cmd/gateway/filter/sample.yaml b/cmd/gateway/filter/sample.yaml index 70e5fa3e30d..e04ac9ab3f4 100644 --- a/cmd/gateway/filter/sample.yaml +++ b/cmd/gateway/filter/sample.yaml @@ -98,7 +98,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/cmd/gateway/lb/sample.yaml b/cmd/gateway/lb/sample.yaml index 129795925b7..f8b471f144d 100644 --- a/cmd/gateway/lb/sample.yaml +++ b/cmd/gateway/lb/sample.yaml @@ -14,7 +14,7 @@ # limitations under the License. # version: v0.0.0 -time_zone: JST +time_zone: UTC logging: format: raw level: debug @@ -27,24 +27,41 @@ server_config: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true + enable_reflection: true header_table_size: 0 - initial_conn_window_size: 0 - initial_window_size: 0 - interceptors: [] + initial_conn_window_size: 2097152 + initial_window_size: 1048576 + interceptors: + - RecoverInterceptor keepalive: max_conn_age: "" max_conn_age_grace: "" max_conn_idle: "" - time: "" - timeout: "" + min_time: 10m + permit_without_stream: false + time: 3h + timeout: 60s max_header_list_size: 0 max_receive_message_size: 0 max_send_message_size: 0 read_buffer_size: 0 write_buffer_size: 0 mode: GRPC + network: tcp probe_wait_time: 3s restart: true + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" health_check_servers: - name: liveness host: 0.0.0.0 @@ -57,7 +74,19 @@ server_config: shutdown_duration: 5s write_timeout: "" mode: "" + network: tcp probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" - name: readiness host: 0.0.0.0 port: 3001 @@ -69,58 +98,99 @@ server_config: shutdown_duration: 0s write_timeout: "" mode: "" + network: tcp probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" metrics_servers: + - name: pprof + host: 0.0.0.0 + port: 6060 + http: + handler_timeout: 5s + idle_timeout: 2s + read_header_timeout: 1s + read_timeout: 1s + shutdown_duration: 5s + write_timeout: 1m + mode: REST + network: tcp + probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: true + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" startup_strategy: - liveness + - pprof - grpc - readiness + shutdown_strategy: + - readiness + - grpc + - pprof + - liveness full_shutdown_duration: 600s tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key observability: enabled: false - collector: - duration: 5s - metrics: - enable_cgo: true - enable_goroutine: true - enable_memory: true - enable_version_info: true - version_info_labels: - - vald_version - - server_name - - git_commit - - build_time - - go_version - - go_os - - go_arch - - ngt_version + otlp: + collector_endpoint: "" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-lb-gateway" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - algorithm_info trace: enabled: false - sampling_rate: 1 - prometheus: - enabled: false - endpoint: /metrics - namespace: vald - jaeger: - enabled: false - collector_endpoint: "" - agent_endpoint: "jaeger-agent.default.svc.cluster.local:6831" - username: "" - password: "" - service_name: "vald-lb-gateway" - buffer_max_count: 10 gateway: agent_port: 8081 - agent_name: "vald-agent-ngt" - agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_name: "vald-agent" + agent_dns: vald-agent.default.svc.cluster.local agent_namespace: "_MY_POD_NAMESPACE_" node_name: "" - index_replica: 5 + index_replica: 3 discoverer: duration: 200ms client: @@ -130,7 +200,7 @@ gateway: connection_pool: enable_dns_resolver: true enable_rebalance: true - old_conn_close_duration: 3s + old_conn_close_duration: 2m rebalance_duration: 30m size: 3 backoff: @@ -141,6 +211,12 @@ gateway: jitter_limit: 100ms maximum_duration: 5s retry_count: 100 + circuit_breaker: + closed_error_rate: 0.7 + closed_refresh_timeout: 10s + half_open_error_rate: 0.5 + min_samples: 1000 + open_timeout: 1s call_option: max_recv_msg_size: 0 max_retry_rpc_buffer_size: 0 @@ -152,17 +228,17 @@ gateway: backoff_max_delay: 120s backoff_multiplier: 1.6 enable_backoff: false - initial_connection_window_size: 0 - initial_window_size: 0 + initial_connection_window_size: 2097152 + initial_window_size: 1048576 insecure: true + interceptors: [] keepalive: permit_without_stream: false time: "" - timeout: "" + timeout: 30s max_msg_size: 0 min_connection_timeout: 20s - read_buffer_size: 0 - tcp: + net: dialer: dual_stack_enabled: true keepalive: "" @@ -171,17 +247,30 @@ gateway: cache_enabled: true cache_expiration: 1h refresh_duration: 30m + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key + read_buffer_size: 0 timeout: "" write_buffer_size: 0 tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key agent_client_options: addrs: [] @@ -189,7 +278,7 @@ gateway: connection_pool: enable_dns_resolver: true enable_rebalance: true - old_conn_close_duration: 3s + old_conn_close_duration: 2m rebalance_duration: 30m size: 3 backoff: @@ -200,6 +289,12 @@ gateway: jitter_limit: 100ms maximum_duration: 5s retry_count: 100 + circuit_breaker: + closed_error_rate: 0.7 + closed_refresh_timeout: 10s + half_open_error_rate: 0.5 + min_samples: 1000 + open_timeout: 1s call_option: max_recv_msg_size: 0 max_retry_rpc_buffer_size: 0 @@ -211,17 +306,17 @@ gateway: backoff_max_delay: 120s backoff_multiplier: 1.6 enable_backoff: false - initial_connection_window_size: 0 - initial_window_size: 0 + initial_connection_window_size: 2097152 + initial_window_size: 1048576 insecure: true + interceptors: [] keepalive: permit_without_stream: false time: "" - timeout: "" + timeout: 30s max_msg_size: 0 min_connection_timeout: 20s - read_buffer_size: 0 - tcp: + net: dialer: dual_stack_enabled: true keepalive: "" @@ -230,15 +325,28 @@ gateway: cache_enabled: true cache_expiration: 1h refresh_duration: 30m + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key + read_buffer_size: 0 timeout: "" write_buffer_size: 0 tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key diff --git a/cmd/manager/index/sample.yaml b/cmd/manager/index/sample.yaml index 3481b80d642..24d3b75836d 100644 --- a/cmd/manager/index/sample.yaml +++ b/cmd/manager/index/sample.yaml @@ -14,7 +14,7 @@ # limitations under the License. # version: v0.0.0 -time_zone: JST +time_zone: UTC logging: format: raw level: debug @@ -27,24 +27,41 @@ server_config: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true + enable_reflection: true header_table_size: 0 - initial_conn_window_size: 0 - initial_window_size: 0 - interceptors: [] + initial_conn_window_size: 2097152 + initial_window_size: 1048576 + interceptors: + - RecoverInterceptor keepalive: max_conn_age: "" max_conn_age_grace: "" max_conn_idle: "" - time: "" - timeout: "" + min_time: 10m + permit_without_stream: false + time: 3h + timeout: 60s max_header_list_size: 0 max_receive_message_size: 0 max_send_message_size: 0 read_buffer_size: 0 write_buffer_size: 0 mode: GRPC + network: tcp probe_wait_time: 3s restart: true + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" health_check_servers: - name: liveness host: 0.0.0.0 @@ -57,7 +74,19 @@ server_config: shutdown_duration: 5s write_timeout: "" mode: "" + network: tcp probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" - name: readiness host: 0.0.0.0 port: 3001 @@ -69,55 +98,96 @@ server_config: shutdown_duration: 0s write_timeout: "" mode: "" + network: tcp probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: true + tcp_no_delay: true + tcp_quick_ack: true + socket_path: "" metrics_servers: + - name: pprof + host: 0.0.0.0 + port: 6060 + http: + handler_timeout: 5s + idle_timeout: 2s + read_header_timeout: 1s + read_timeout: 1s + shutdown_duration: 5s + write_timeout: 1m + mode: REST + network: tcp + probe_wait_time: 3s + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: true + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" startup_strategy: - liveness + - pprof - grpc - readiness + shutdown_strategy: + - readiness + - grpc + - pprof + - liveness full_shutdown_duration: 600s tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key observability: enabled: false - collector: - duration: 5s - metrics: - enable_cgo: true - enable_goroutine: true - enable_memory: true - enable_version_info: true - version_info_labels: - - vald_version - - server_name - - git_commit - - build_time - - go_version - - go_os - - go_arch - - ngt_version + otlp: + collector_endpoint: "" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-manager-index" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - algorithm_info trace: enabled: false - sampling_rate: 1 - prometheus: - enabled: false - endpoint: /metrics - namespace: vald - jaeger: - enabled: false - collector_endpoint: "" - agent_endpoint: "jaeger-agent.default.svc.cluster.local:6831" - username: "" - password: "" - service_name: "vald-manager-index" - buffer_max_count: 10 indexer: agent_port: 8081 - agent_name: "vald-agent-ngt" - agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_name: "vald-agent" + agent_dns: vald-agent.default.svc.cluster.local agent_namespace: "_MY_POD_NAMESPACE_" node_name: "" discoverer: @@ -129,7 +199,7 @@ indexer: connection_pool: enable_dns_resolver: true enable_rebalance: true - old_conn_close_duration: 3s + old_conn_close_duration: 2m rebalance_duration: 30m size: 3 backoff: @@ -140,6 +210,12 @@ indexer: jitter_limit: 100ms maximum_duration: 5s retry_count: 100 + circuit_breaker: + closed_error_rate: 0.7 + closed_refresh_timeout: 10s + half_open_error_rate: 0.5 + min_samples: 1000 + open_timeout: 1s call_option: max_recv_msg_size: 0 max_retry_rpc_buffer_size: 0 @@ -151,17 +227,17 @@ indexer: backoff_max_delay: 120s backoff_multiplier: 1.6 enable_backoff: false - initial_connection_window_size: 0 - initial_window_size: 0 + initial_connection_window_size: 2097152 + initial_window_size: 1048576 insecure: true + interceptors: [] keepalive: permit_without_stream: false time: "" - timeout: "" + timeout: 30s max_msg_size: 0 min_connection_timeout: 20s - read_buffer_size: 0 - tcp: + net: dialer: dual_stack_enabled: true keepalive: "" @@ -170,17 +246,30 @@ indexer: cache_enabled: true cache_expiration: 1h refresh_duration: 30m + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key + read_buffer_size: 0 timeout: "" write_buffer_size: 0 tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key agent_client_options: addrs: [] @@ -188,7 +277,7 @@ indexer: connection_pool: enable_dns_resolver: true enable_rebalance: true - old_conn_close_duration: 3s + old_conn_close_duration: 2m rebalance_duration: 30m size: 3 backoff: @@ -199,6 +288,12 @@ indexer: jitter_limit: 100ms maximum_duration: 5s retry_count: 100 + circuit_breaker: + closed_error_rate: 0.7 + closed_refresh_timeout: 10s + half_open_error_rate: 0.5 + min_samples: 1000 + open_timeout: 1s call_option: max_recv_msg_size: 0 max_retry_rpc_buffer_size: 0 @@ -207,8 +302,8 @@ indexer: dial_option: write_buffer_size: 0 read_buffer_size: 0 - initial_window_size: 0 - initial_connection_window_size: 0 + initial_window_size: 1.048576e+06 + initial_connection_window_size: 2.097152e+06 max_msg_size: 0 backoff_max_delay: "120s" backoff_base_delay: "1s" @@ -218,7 +313,8 @@ indexer: enable_backoff: false insecure: true timeout: "" - tcp: + interceptors: [] + net: dns: cache_enabled: true cache_expiration: 1h @@ -231,18 +327,32 @@ indexer: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false keepalive: permit_without_stream: false time: "" - timeout: "" + timeout: 30s tls: ca: /path/to/ca cert: /path/to/cert enabled: false + insecure_skip_verify: false key: /path/to/key concurrency: 1 auto_index_duration_limit: 30m auto_index_check_duration: 1m + auto_save_index_duration_limit: 3h + auto_save_index_wait_duration: 10m auto_index_length: 100 creation_pool_size: 10000 diff --git a/cmd/tools/cli/loadtest/sample.yaml b/cmd/tools/cli/loadtest/sample.yaml index 11a121542fe..500b87d2771 100644 --- a/cmd/tools/cli/loadtest/sample.yaml +++ b/cmd/tools/cli/loadtest/sample.yaml @@ -13,20 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # - ---- version: v0.0.0 time_zone: JST logging: logger: glg - level: info - format: raw -operation: insert + level: debug + format: json +method: insert dataset: fashion-mnist concurrency: 100 -batch_size: 10 -progress_duration: 3s -addr: "localhost:8082" +addr: "localhost:8081" client: addrs: [] health_check_duration: 1s @@ -43,8 +39,6 @@ client: max_retry_rpc_buffer_size: 0 max_send_msg_size: 0 wait_for_ready: true - connection_pool: - enable_dns_resolver: true dial_option: enable_backoff: false initial_connection_window_size: 0 diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile new file mode 100644 index 00000000000..53c25c6649b --- /dev/null +++ b/dockers/agent/core/faiss/Dockerfile @@ -0,0 +1,120 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG GO_VERSION=latest +ARG MAINTAINER="vdaas.org vald team " + +FROM golang:${GO_VERSION} AS golang + +FROM ubuntu:devel AS builder + +ENV GO111MODULE on +ENV DEBIAN_FRONTEND noninteractive +ENV INITRD No +ENV LANG en_US.UTF-8 +ENV GOROOT /opt/go +ENV GOPATH /go +ENV PATH ${PATH}:${GOROOT}/bin:${GOPATH}/bin +ENV ORG vdaas +ENV REPO vald +ENV PKG agent/core/faiss +ENV PKG_INTERNAL agent/internal +ENV APP_NAME faiss + +# skipcq: DOK-DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + gcc \ + git \ + g++ \ + intel-mkl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=golang /usr/local/go $GOROOT +RUN mkdir $GOPATH + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} + +COPY go.mod . +COPY go.sum . + +RUN go mod download + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal +COPY internal . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc +COPY apis/grpc . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} +COPY pkg/${PKG} . +# copy ngt/service/kvs and ngt/service/vqueue +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/agent/core/ngt/service +COPY pkg/agent/core/ngt/service/kvs ./kvs +COPY pkg/agent/core/ngt/service/vqueue ./vqueue + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG_INTERNAL} +COPY pkg/${PKG_INTERNAL} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +COPY cmd/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY Makefile . +RUN update-alternatives --set libblas.so-x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/libmkl_rt.so \ + && make faiss/install + +COPY .git . + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +RUN cp sample.yaml /tmp/config.yaml + +FROM ubuntu:devel +LABEL maintainer "${MAINTAINER}" + +ENV APP_NAME faiss + +COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +COPY --from=builder /tmp/config.yaml /etc/server/config.yaml + +COPY --from=builder /usr/local/lib/libfaiss.so /usr/local/lib/libfaiss.so +COPY --from=builder /lib/x86_64-linux-gnu/libstdc++.so.6 /lib/x86_64-linux-gnu/libstdc++.so.6 +COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/x86_64-linux-gnu/libgcc_s.so.1 +COPY --from=builder /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/libc.so.6 +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_intel_lp64.so /lib/x86_64-linux-gnu/libmkl_intel_lp64.so +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_sequential.so /lib/x86_64-linux-gnu/libmkl_sequential.so +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_core.so /lib/x86_64-linux-gnu/libmkl_core.so +COPY --from=builder /lib/x86_64-linux-gnu/libgomp.so.1 /lib/x86_64-linux-gnu/libgomp.so.1 +COPY --from=builder /lib/x86_64-linux-gnu/libm.so.6 /lib/x86_64-linux-gnu/libm.so.6 +COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/x86_64-linux-gnu/libdl.so.2 +COPY --from=builder /lib/x86_64-linux-gnu/libpthread.so.0 /lib/x86_64-linux-gnu/libpthread.so.0 +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_avx2.so /lib/x86_64-linux-gnu/libmkl_avx2.so +RUN ldconfig -v + +ENTRYPOINT ["/go/bin/faiss"] diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index aacc18519f1..c5c5c9bed9d 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -70,6 +70,7 @@ COPY Makefile.d Makefile.d COPY apis/proto apis/proto COPY versions versions COPY hack/go.mod.default hack/go.mod.default +COPY example example RUN make deps ROOTDIR=$ROOTDIR diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index f27aeb331ea..28fa7f18d75 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -49,6 +49,7 @@ COPY Makefile . COPY Makefile.d Makefile.d COPY versions versions COPY hack/go.mod.default hack/go.mod.default +COPY example example # basic deps RUN make deps \ diff --git a/docs/overview/component/README.md b/docs/overview/component/README.md index 49461c6b9bb..58e03e547e5 100644 --- a/docs/overview/component/README.md +++ b/docs/overview/component/README.md @@ -67,6 +67,7 @@ In this section, we will describe what is Vald Agent and the corresponding compo Vald Agent provides functionalities to perform approximate nearest neighbor search. Agent-NGT uses [yahoojapan/NGT](https://github.com/yahoojapan/NGT) as a core library. +And Agent-Faiss uses [facebookresearch/faiss](https://github.com/facebookresearch/faiss) as a core library. Each Vald Agent pod has its own vector data space because only several Vald Agents are selected to be inserted/updated in a single insert/update request. diff --git a/docs/overview/component/agent.md b/docs/overview/component/agent.md index be0326bb7f7..1cf33ad5348 100644 --- a/docs/overview/component/agent.md +++ b/docs/overview/component/agent.md @@ -78,6 +78,56 @@ This image shows the mechanism to create NGT index. Please refer to [Go Doc](https://pkg.go.dev/github.com/vdaas/vald@v1.3.1/pkg/agent/core/ngt/service) for other functions. +#### Vald Agent Faiss + +Vald Agent Faiss uses [Faiss](https://github.com/facebookresearch/faiss) as an algorithm. + +The main functions are the followings: + +- Insert + - Request to insert new vectors into the Faiss. + - Requested vectors are stored in the `vqueue`. + - Cache a fixed number of verctors for Faiss training. + - Once Faiss trained in CreateIndex, the vector is never cached for Faiss training. +- Search + - Get the nearest neighbor vectors of the request vector from Faiss indexes. + - radius/epsilon is search config for NGT and has no meaning in Faiss. +- Update + - Create a request to update the specific vectors to the new vectors. + - Requested vectors are stored in the `vqueue`. +- Remove + - Create a request to remove the specific vectors from Faiss indexes. + - Requested vectors are stored in the `vqueue`. +- Exist + - Check whether the specific vectors are already inserted or not. +- CreateIndex + - Create a new Faiss index structure in memory using vectors stored in the `vqueue` and the existing Faiss index structure if it exists. + - If a certain number of vectors required for Faiss training are not cached, they will not be trained. + - If Faiss is not trained, no index is generated. +- SaveIndex + - Save metadata about Faiss index information to the internal storage. + +Unimplemented functions are the followings: + +- GetObject +- SearchByID +- StreamXXX +- MultiXXX + +
+Same as Agent NGT, You have to control the duration of CreateIndex and SaveIndex by configuration. + +These methods don’t always run when getting the request. + +
+ +
+As you see, Vald Agent Faiss can only search the nearest neighbors from the Faiss index. + +You have to wait to complete the CreateIndex and SaveIndex functions before searching. + +
+ ### Sidecar `Sidecar` saves the index metadata file to external storage like Amazon S3 or Google Cloud Storage. diff --git a/docs/tutorial/get-started-with-faiss-agent.md b/docs/tutorial/get-started-with-faiss-agent.md new file mode 100644 index 00000000000..87d75621601 --- /dev/null +++ b/docs/tutorial/get-started-with-faiss-agent.md @@ -0,0 +1,393 @@ +# Get Started + +This tutorial is for those who have already completed [Get Started](https://github.com/vdaas/vald/blob/main/docs/tutorial/get-started.md). +Please refer to Prepare the Kubernetes Cluster and others there. + +## Deploy Vald on Kubernetes Cluster + +This chapter shows how to deploy Vald using Helm and run it on your Kubernetes cluster.
+In this tutorial, you will deploy the basic configuration of Vald that is consisted of vald-agent-faiss, vald-lb-gateway, vald-discoverer, and vald-manager-index.
+ +1. Clone the repository + + ```bash + git clone https://github.com/vdaas/vald.git && \ + cd vald + ``` + +1. Confirm which cluster to deploy + + ```bash + kubectl cluster-info + ``` + +1. Edit Configurations + + Set the parameters for connecting to the vald-lb-gateway through Kubernetes ingress from the external network. + Please set these parameters. + + ```bash + vim example/helm/values.yaml + === + ## vald-lb-gateway settings + gateway: + lb: + ... + ingress: + enabled: true + # TODO: Set your ingress host. + host: localhost + # TODO: Set annotations which you have to set for your k8s cluster. + annotations: + ... + ## vald-agent-faiss settings + agent: + algorithm: faiss + image: + repository: vdaas/vald-agent-faiss + tag: latest + faiss: + auto_index_check_duration: 1m + auto_index_duration_limit: 24h + auto_index_length: 10 + auto_save_index_duration: 35m + dimension: 784 + enable_copy_on_write: false + enable_in_memory_mode: true + enable_proactive_gc: true + index_path: "" + initial_delay_max_duration: 3m + load_index_timeout_factor: 1ms + m: 8 # dimension % m == 0, train size >= 2^m(or nlist) * minPointsPerCentroid + max_load_index_timeout: 10m + metric_type: "inner_product" + min_load_index_timeout: 3m + nbits_per_idx: 8 + nlist: 100 + ... + ``` + + Note:
+ If you decided to use port-forward instead of ingress, please set `gateway.lb.ingress.enabled` to `false`. + +1. Deploy Vald using Helm + + Add vald repo into the helm repo. + + ```bash + helm repo add vald https://vald.vdaas.org/charts + ``` + + Deploy vald on your Kubernetes cluster. + + ```bash + helm install vald vald/vald --values example/helm/values.yaml + ``` + +1. Verify + + When finish deploying Vald, you can check the Vald's pods status following command. + + ```bash + kubectl get pods + ``` + +
Example output
+ If the deployment is successful, all Vald components should be running. + + ```bash + NAME READY STATUS RESTARTS AGE + vald-agent-faiss-0 1/1 Running 0 7m12s + vald-agent-faiss-1 1/1 Running 0 7m12s + vald-agent-faiss-2 1/1 Running 0 7m12s + vald-agent-faiss-3 1/1 Running 0 7m12s + vald-agent-faiss-4 1/1 Running 0 7m12s + vald-discoverer-7f9f697dbb-q44qh 1/1 Running 0 7m11s + vald-lb-gateway-6b7b9f6948-4z5md 1/1 Running 0 7m12s + vald-lb-gateway-6b7b9f6948-68g94 1/1 Running 0 6m56s + vald-lb-gateway-6b7b9f6948-cvspq 1/1 Running 0 6m56s + vald-manager-index-74c7b5ddd6-jrnlw 1/1 Running 0 7m12s + ``` + +
+ + ```bash + kubectl get ingress + ``` + +
Example output
+ + ```bash + NAME CLASS HOSTS ADDRESS PORTS AGE + vald-lb-gateway-ingress localhost 192.168.16.2 80 7m43s + ``` + +
+ + ```bash + kubectl get svc + ``` + +
Example output
+ + ```bash + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kubernetes ClusterIP 10.43.0.1 443/TCP 9m29s + vald-agent-faiss ClusterIP None 8081/TCP,3001/TCP 8m48s + vald-discoverer ClusterIP None 8081/TCP,3001/TCP 8m48s + vald-manager-index ClusterIP None 8081/TCP,3001/TCP 8m48s + vald-lb-gateway ClusterIP None 8081/TCP,3001/TCP 8m48s + ``` + +
+ +## Run Example Code + +In this chapter, you will execute insert, search, and delete vectors to your Vald cluster using the example code.
+The [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) is used as a dataset for indexing and search query. + +The example code is implemented in Go and using [vald-client-go](https://github.com/vdaas/vald-client-go), one of the official Vald client libraries, for requesting to Vald cluster. +Vald provides multiple language client libraries such as Go, Java, Node.js, Python, etc. +If you are interested, please refer to [SDKs](../user-guides/sdks.md).
+ +1. Port Forward(option) + + If you do not use Kubernetes Ingress, port-forward is required to make requests from your local environment. + + ```bash + kubectl port-forward deployment/vald-lb-gateway 8081:8081 + ``` + +1. Download dataset + + Download [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) that is used as a dataset for indexing and search query. + + Move to the working directory + + ```bash + cd example/client + ``` + + Download Fashion-MNIST testing dataset + + ```bash + wget http://ann-benchmarks.com/fashion-mnist-784-euclidean.hdf5 + ``` + +1. Run Example + + We use [`example/client/main.go`](https://github.com/vdaas/vald/blob/main/example/client/main.go) to run the example.
+ This example will insert and index 400 vectors into the Vald from the Fashion-MNIST dataset via [gRPC](https://grpc.io/). + And then after waiting for indexing, it will request for searching the nearest vector 10 times. + You will get the 10 nearest neighbor vectors for each search query.
+ Run example codes by executing the below command. + + ```bash + go run main.go + ``` + +
The detailed explanation of example code is here
+ This will execute 6 steps. + + 1. init + + - Import packages +
example code
+ + ```go + package main + + import ( + "context" + "encoding/json" + "flag" + "time" + + "github.com/kpango/fuid" + "github.com/kpango/glg" + "github.com/vdaas/vald-client-go/v1/payload" + "github.com/vdaas/vald-client-go/v1/vald" + + "gonum.org/v1/hdf5" + "google.golang.org/grpc" + ) + ``` + +
+ + - Set variables + + - The constant number of training datasets and test datasets. +
example code
+ + ```go + const ( + insertCount = 400 + testCount = 20 + ) + ``` + +
+ + - The variables for configuration. +
example code
+ + ```go + const ( + datasetPath string + grpcServerAddr string + indexingWaitSeconds uint + ) + ``` + +
+ + - Recognition parameters. +
example code
+ + ```go + func init() { + flag.StringVar(&datasetPath, "path", "fashion-mnist-784-euclidean.hdf5", "set dataset path") + flag.StringVar(&grpcServerAddr, "addr", "127.0.0.1:8081", "set gRPC server address") + flag.UintVar(&indexingWaitSeconds, "wait", 60, "set indexing wait seconds") + flag.Parse() + } + ``` + +
+ + 1. load + + - Loading from Fashion-MNIST dataset and set id for each vector that is loaded. This step will return the training dataset, test dataset, and ids list of ids when loading is completed with success. +
example code
+ + ```go + ids, train, test, err := load(datasetPath) + if err != nil { + glg.Fatal(err) + } + ``` + +
+ + 1. Create the gRPC connection and Vald client with gRPC connection. + +
example code
+ + ```go + ctx := context.Background() + + conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithInsecure()) + if err != nil { + glg.Fatal(err) + } + + client := vald.NewValdClient(conn) + ``` + +
+ + 1. Insert and Index + + - Insert and Indexing 400 training datasets to the Vald agent. +
example code
+ + ```go + for i := range ids [:insertCount] { + _, err := client.Insert(ctx, &payload.Insert_Request{ + Vector: &payload.Object_Vector{ + Id: ids[i], + Vector: train[i], + }, + Config: &payload.Insert_Config{ + SkipStrictExistCheck: true, + }, + }) + if err != nil { + glg.Fatal(err) + } + if i%10 == 0 { + glg.Infof("Inserted %d", i) + } + } + ``` + +
+ + - Wait until indexing finish. +
example code
+ + ```go + wt := time.Duration(indexingWaitSeconds) * time.Second + glg.Infof("Wait %s for indexing to finish", wt) + time.Sleep(wt) + ``` + +
+ + 1. Search + + - Search 10 neighbor vectors for each 20 test datasets and return a list of the neighbor vectors. + + - When getting approximate vectors, the Vald client sends search config and vector to the server via gRPC. +
example code
+ + ```go + glg.Infof("Start search %d times", testCount) + for i, vec := range test[:testCount] { + res, err := client.Search(ctx, &payload.Search_Request){ + Vector: vec, + Config: &payload.Search_Config{ + Num: 10, + Radius: -1, + Epsilon: 0.1, + Timeout: 100000000, + } + } + if err != nil { + glg.Fatal(err) + } + + b, _ := json.MarshalIndent(res.GetResults(), "", " ") + glg.Infof("%d - Results : %s\n\n", i+1, string(b)) + time.Sleep(1 * time.Second) + } + ``` + +
+ + 1. Remove + + - Remove 400 indexed training datasets from the Vald agent. +
example code
+ + ```go + for i := range ids [:insertCount] { + _, err := client.Remove(ctx, &payload.Remove_Request{ + Id: &payload.Object_ID{ + Id: ids[i], + }, + }) + if err != nil { + glg.Fatal(err) + } + if i%10 == 0 { + glg.Infof("Removed %d", i) + } + } + ``` + +
+ +## Cleanup + +In the last, you can remove the deployed Vald Cluster by executing the below command. + +```bash +helm uninstall vald +``` + +## References + +- [Get Started with NGT agent by default](https://github.com/vdaas/vald/blob/main/docs/tutorial/get-started.md) +- [Faiss](https://github.com/facebookresearch/faiss) diff --git a/docs/user-guides/backup-configuration.md b/docs/user-guides/backup-configuration.md index 0f8872f38f8..06c1e3b21ab 100644 --- a/docs/user-guides/backup-configuration.md +++ b/docs/user-guides/backup-configuration.md @@ -29,7 +29,7 @@ Please refer to the following tables and decide which method fit for your case. This section shows the best practice for configuring backup features with PV, S3, or PV + S3. -Each sample configuration yaml is published on [here](https://github.com/vdaas/vald/tree/master/charts/vald/values). +Each sample configuration yaml is published on [here](https://github.com/vdaas/vald/tree/main/charts/vald/values). Please refer it for more details. ### General diff --git a/docs/user-guides/configuration.md b/docs/user-guides/configuration.md index b3b9e06d882..f54def2a171 100644 --- a/docs/user-guides/configuration.md +++ b/docs/user-guides/configuration.md @@ -175,6 +175,42 @@ When the setting parameter of Vald Agent NGT is shorter than the setting value o If this happens, the Index Manager may not function properly. +#### Faiss + +Vald Agent Faiss uses [facebookresearch/faiss][faiss] as a core library for searching vectors. +The behaviors of Faiss can be configured by setting `agent.faiss` field object. + +The important parameters are the followings: + +- `agent.faiss.dimension` +- `agent.faiss.distance_type` +- `agent.faiss.m` +- `agent.faiss.metric_type` +- `agent.faiss.nbits_per_idx` +- `agent.faiss.nlist` + +Users should configure these parameters first to fit their use case. +For further details, please read [the Fiass wiki][faiss-wiki]. + +Vald Agent Faiss has a feature to start indexing automatically. +The behavior of this feature can be configured with these parameters: + +- `agent.faiss.auto_index_duration_limit` +- `agent.faiss.auto_index_check_duration` +- `agent.faiss.auto_index_length` + +
+While the Vald Agent Faiss is in the process of creating indexes, it will ignore all search requests to the target pods. +
+ +
+When deploying Vald Index Manager, the above parameters should be set much longer than the Vald Index Manager settings (Please refer to the Vald Index Manager section) or minus value.
+E.g., set agent.faiss.auto_index_duration_limit to "720h" or "-1h" and agent.faiss.auto_index_check_duration to "24h" or "-1h".
+This is because the Vald Index Manager accurately grasps the index information of each Vald Agent Faiss and controls the execution timing of indexing.

+When the setting parameter of Vald Agent Faiss is shorter than the setting value of Vald Index Manager, Vald Agent Faiss may start indexing by itself without the execution command from Vald Index Manager. +If this happens, the Index Manager may not function properly. +
+ #### Resource requests and limits, Pod priorities Because the Vald Agent pod places indexes on memory, termination of agent pods causes loss of indexes. @@ -371,3 +407,5 @@ For further details, there are references to the Helm values in the Vald GitHub [kubernetes-topology-spread-constraints]: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ [yj-ngt]: https://github.com/yahoojapan/NGT [yj-ngt-wiki]: https://github.com/yahoojapan/NGT/wiki +[faiss]: https://github.com/facebookresearch/faiss +[faiss-wiki]: https://github.com/facebookresearch/faiss/wiki diff --git a/docs/user-guides/observability-configuration.md b/docs/user-guides/observability-configuration.md index cf2734ab9e2..66617e874b2 100644 --- a/docs/user-guides/observability-configuration.md +++ b/docs/user-guides/observability-configuration.md @@ -132,7 +132,7 @@ defaults: - "go_version" - "go_os" - "go_arch" - - "ngt_version" + - "algorithm_info" # enable memory metrics enable_memory: true # enable goroutine metrics diff --git a/example/client/go.mod b/example/client/go.mod index a356acea235..ff62b83e7e2 100644 --- a/example/client/go.mod +++ b/example/client/go.mod @@ -7,6 +7,7 @@ replace ( github.com/goccy/go-json => github.com/goccy/go-json v0.10.2 github.com/golang/protobuf => github.com/golang/protobuf v1.5.3 github.com/kpango/glg => github.com/kpango/glg v1.6.15 +<<<<<<< HEAD github.com/pkg/sftp => github.com/pkg/sftp v1.13.6 golang.org/x/crypto => golang.org/x/crypto v0.14.0 golang.org/x/net => golang.org/x/net v0.17.0 @@ -15,6 +16,16 @@ replace ( google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b google.golang.org/grpc => google.golang.org/grpc v1.58.3 +======= + github.com/pkg/sftp => github.com/pkg/sftp v1.13.5 + golang.org/x/crypto => golang.org/x/crypto v0.11.0 + golang.org/x/net => golang.org/x/net v0.12.0 + golang.org/x/text => golang.org/x/text v0.11.0 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 + google.golang.org/grpc => google.golang.org/grpc v1.56.2 +>>>>>>> feature/agent/new-agents-helm-template-base google.golang.org/protobuf => google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1 @@ -25,7 +36,11 @@ require ( github.com/kpango/glg v1.6.14 github.com/vdaas/vald-client-go v1.7.8 gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 +<<<<<<< HEAD google.golang.org/grpc v1.58.3 +======= + google.golang.org/grpc v1.56.2 +>>>>>>> feature/agent/new-agents-helm-template-base ) require ( @@ -33,6 +48,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/kpango/fastime v1.1.9 // indirect +<<<<<<< HEAD github.com/planetscale/vtprotobuf v0.5.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect @@ -40,5 +56,13 @@ require ( google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect +======= + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect +>>>>>>> feature/agent/new-agents-helm-template-base google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/example/client/go.sum b/example/client/go.sum index b30397075d7..56e0fc21ee4 100644 --- a/example/client/go.sum +++ b/example/client/go.sum @@ -28,6 +28,7 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +<<<<<<< HEAD go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -35,10 +36,19 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +======= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +>>>>>>> feature/agent/new-agents-helm-template-base golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= +<<<<<<< HEAD google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= @@ -47,6 +57,16 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +======= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +>>>>>>> feature/agent/new-agents-helm-template-base google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/example/client/main.go b/example/client/main.go index 386de999174..e91ea745ee6 100644 --- a/example/client/main.go +++ b/example/client/main.go @@ -29,13 +29,13 @@ import ( ) const ( - insertCount = 400 - testCount = 20 + testCount = 20 ) var ( datasetPath string grpcServerAddr string + insertCount uint indexingWaitSeconds uint ) @@ -43,10 +43,19 @@ func init() { /** Path option specifies hdf file by path. Default value is `fashion-mnist-784-euclidean.hdf5`. Addr option specifies grpc server address. Default value is `127.0.0.1:8081`. +<<<<<<< HEAD Wait option specifies indexing wait time (in seconds). Default value is `60`. **/ flag.StringVar(&datasetPath, "path", "fashion-mnist-784-euclidean.hdf5", "dataset path") flag.StringVar(&grpcServerAddr, "addr", "localhost:8081", "gRPC server address") +======= + Insert option specifies insert count. Default value is `400`. + Wait option specifies indexing wait time (in seconds). Default value is `60`. + **/ + flag.StringVar(&datasetPath, "path", "fashion-mnist-784-euclidean.hdf5", "dataset path") + flag.StringVar(&grpcServerAddr, "addr", "localhost:8081", "gRPC server address") + flag.UintVar(&insertCount, "insert", 400, "insert count") +>>>>>>> feature/agent/new-agents-helm-template-base flag.UintVar(&indexingWaitSeconds, "wait", 60, "indexing wait seconds") flag.Parse() } diff --git a/go.mod b/go.mod index 8496307df8b..d456f210b5f 100755 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/vdaas/vald go 1.21 replace ( +<<<<<<< HEAD cloud.google.com/go => cloud.google.com/go v0.111.0 cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.57.1 cloud.google.com/go/compute => cloud.google.com/go/compute v1.23.3 @@ -16,18 +17,42 @@ replace ( cloud.google.com/go/storage => cloud.google.com/go/storage v1.35.1 cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.4 code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20231017140541-3b893ed0421b +======= + cloud.google.com/go => cloud.google.com/go v0.110.6 + cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.52.0 + cloud.google.com/go/compute => cloud.google.com/go/compute v1.21.0 + cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.12.1 + cloud.google.com/go/firestore => cloud.google.com/go/firestore v1.11.0 + cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.1 + cloud.google.com/go/kms => cloud.google.com/go/kms v1.13.0 + cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.15.1 + cloud.google.com/go/pubsub => cloud.google.com/go/pubsub v1.32.0 + cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.11.1 + cloud.google.com/go/storage => cloud.google.com/go/storage v1.31.0 + cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.1 + code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20230612151507-41ef4d1f67a4 +>>>>>>> feature/agent/new-agents-helm-template-base contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws v0.0.0-20230502192102-15967c811cec contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus v0.4.2 contrib.go.opencensus.io/integrations/ocsql => contrib.go.opencensus.io/integrations/ocsql v0.1.7 git.sr.ht/~sbinet/gg => git.sr.ht/~sbinet/gg v0.5.0 github.com/Azure/azure-amqp-common-go/v3 => github.com/Azure/azure-amqp-common-go/v3 v3.2.3 github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible +<<<<<<< HEAD github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 github.com/Azure/go-amqp => github.com/Azure/go-amqp v1.0.2 github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.1-0.20230905222633-df94ce56f001+incompatible github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/autorest v0.11.30-0.20230905222633-df94ce56f001 +======= + github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 + github.com/Azure/go-amqp => github.com/Azure/go-amqp v1.0.1 + github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.1-0.20230628134501-d3f8f8a6cdf2+incompatible + github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/autorest v0.11.30-0.20230628134501-d3f8f8a6cdf2 +>>>>>>> feature/agent/new-agents-helm-template-base github.com/Azure/go-autorest/autorest/adal => github.com/Azure/go-autorest/autorest/adal v0.9.23 github.com/Azure/go-autorest/autorest/date => github.com/Azure/go-autorest/autorest/date v0.3.1-0.20230905222633-df94ce56f001 github.com/Azure/go-autorest/autorest/mocks => github.com/Azure/go-autorest/autorest/mocks v0.4.3-0.20230905222633-df94ce56f001 @@ -38,6 +63,7 @@ replace ( github.com/DATA-DOG/go-sqlmock => github.com/DATA-DOG/go-sqlmock v1.5.1 github.com/GoogleCloudPlatform/cloudsql-proxy => github.com/GoogleCloudPlatform/cloudsql-proxy v1.33.15 github.com/Masterminds/semver/v3 => github.com/Masterminds/semver/v3 v3.2.1 +<<<<<<< HEAD github.com/ajstarks/deck => github.com/ajstarks/deck v0.0.0-20231012031509-f833e437b68a github.com/ajstarks/deck/generate => github.com/ajstarks/deck/generate v0.0.0-20231012031509-f833e437b68a github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b @@ -66,6 +92,36 @@ replace ( github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 github.com/aws/smithy-go => github.com/aws/smithy-go v1.19.0 +======= + github.com/ajstarks/deck => github.com/ajstarks/deck v0.0.0-20230713212537-3862a8c503c2 + github.com/ajstarks/deck/generate => github.com/ajstarks/deck/generate v0.0.0-20230713212537-3862a8c503c2 + github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b + github.com/antihax/optional => github.com/antihax/optional v1.0.0 + github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.300 + github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.19.0 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 + github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.28 + github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.27 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 + github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 + github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 + github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 + github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 + github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms v1.23.1 + github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 + github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.11 + github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.20.14 + github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.23.3 + github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.36.8 + github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 + github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 + github.com/aws/smithy-go => github.com/aws/smithy-go v1.13.5 +>>>>>>> feature/agent/new-agents-helm-template-base github.com/benbjohnson/clock => github.com/benbjohnson/clock v1.3.5 github.com/beorn7/perks => github.com/beorn7/perks v1.0.1 github.com/bmizerany/assert => github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 @@ -112,7 +168,11 @@ replace ( github.com/go-logfmt/logfmt => github.com/go-logfmt/logfmt v0.6.0 github.com/go-logr/logr => github.com/go-logr/logr v1.3.0 github.com/go-logr/stdr => github.com/go-logr/stdr v1.2.2 +<<<<<<< HEAD github.com/go-logr/zapr => github.com/go-logr/zapr v1.3.0 +======= + github.com/go-logr/zapr => github.com/go-logr/zapr v1.2.4 +>>>>>>> feature/agent/new-agents-helm-template-base github.com/go-openapi/jsonpointer => github.com/go-openapi/jsonpointer v0.20.0 github.com/go-openapi/jsonreference => github.com/go-openapi/jsonreference v0.20.2 github.com/go-openapi/swag => github.com/go-openapi/swag v0.22.4 @@ -168,7 +228,11 @@ replace ( github.com/hashicorp/go-uuid => github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-version => github.com/hashicorp/go-version v1.6.0 github.com/iancoleman/strcase => github.com/iancoleman/strcase v0.3.0 +<<<<<<< HEAD github.com/ianlancetaylor/demangle => github.com/ianlancetaylor/demangle v0.0.0-20231023195312-e2daf7ba7156 +======= + github.com/ianlancetaylor/demangle => github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab +>>>>>>> feature/agent/new-agents-helm-template-base github.com/inconshreveable/mousetrap => github.com/inconshreveable/mousetrap v1.1.0 github.com/jackc/chunkreader/v2 => github.com/jackc/chunkreader/v2 v2.0.1 github.com/jackc/pgconn => github.com/jackc/pgconn v1.14.1 @@ -190,8 +254,13 @@ replace ( github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v1.0.0 github.com/kisielk/errcheck => github.com/kisielk/errcheck v1.6.3 github.com/kisielk/gotool => github.com/kisielk/gotool v1.0.0 +<<<<<<< HEAD github.com/klauspost/compress => github.com/klauspost/compress v1.17.5-0.20231209164634-6bf960e5bd5d github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.6 +======= + github.com/klauspost/compress => github.com/klauspost/compress v1.16.8-0.20230714153001-f71e4d343a35 + github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.5 +>>>>>>> feature/agent/new-agents-helm-template-base github.com/kpango/fastime => github.com/kpango/fastime v1.1.9 github.com/kpango/fuid => github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 github.com/kpango/gache/v2 => github.com/kpango/gache/v2 v2.0.9 @@ -273,6 +342,7 @@ replace ( github.com/zeebo/xxh3 => github.com/zeebo/xxh3 v1.0.2 go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.8 go.opencensus.io => go.opencensus.io v0.24.0 +<<<<<<< HEAD go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.11.1 go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 @@ -286,16 +356,40 @@ replace ( go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1 go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v1.0.0 go.starlark.net => go.starlark.net v0.0.0-20231121155337-90ade8b19d09 +======= + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 + go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric => go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 + go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.16.0 + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.39.0 + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v1.0.0 + go.starlark.net => go.starlark.net v0.0.0-20230712173630-2226322290fc +>>>>>>> feature/agent/new-agents-helm-template-base go.uber.org/atomic => go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs => go.uber.org/automaxprocs v1.5.3 go.uber.org/goleak => go.uber.org/goleak v1.3.0 go.uber.org/multierr => go.uber.org/multierr v1.11.0 +<<<<<<< HEAD go.uber.org/zap => go.uber.org/zap v1.26.0 gocloud.dev => gocloud.dev v0.35.0 golang.org/x/crypto => golang.org/x/crypto v0.16.0 golang.org/x/exp => golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20231206192017-f3f8817b8deb golang.org/x/image => golang.org/x/image v0.14.0 +======= + go.uber.org/zap => go.uber.org/zap v1.24.0 + gocloud.dev => gocloud.dev v0.30.0 + golang.org/x/crypto => golang.org/x/crypto v0.11.0 + golang.org/x/exp => golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20230713183714-613f0c0eb8a1 + golang.org/x/image => golang.org/x/image v0.9.0 +>>>>>>> feature/agent/new-agents-helm-template-base golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mobile => golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a golang.org/x/mod => golang.org/x/mod v0.14.0 @@ -311,6 +405,7 @@ replace ( gomodules.xyz/jsonpatch/v2 => gomodules.xyz/jsonpatch/v2 v2.4.0 gonum.org/v1/gonum => gonum.org/v1/gonum v0.14.0 gonum.org/v1/hdf5 => gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 +<<<<<<< HEAD gonum.org/v1/plot => gonum.org/v1/plot v0.14.0 google.golang.org/api => google.golang.org/api v0.153.0 google.golang.org/appengine => google.golang.org/appengine v1.6.8 @@ -318,6 +413,15 @@ replace ( google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20231211222908-989df2bf70f3 google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20231211222908-989df2bf70f3 google.golang.org/grpc => google.golang.org/grpc v1.60.0 +======= + gonum.org/v1/plot => gonum.org/v1/plot v0.13.0 + google.golang.org/api => google.golang.org/api v0.131.0 + google.golang.org/appengine => google.golang.org/appengine v1.6.7 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 + google.golang.org/grpc => google.golang.org/grpc v1.56.2 +>>>>>>> feature/agent/new-agents-helm-template-base google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf => google.golang.org/protobuf v1.31.0 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c @@ -342,8 +446,13 @@ replace ( sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.15.0 sigs.k8s.io/json => sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd sigs.k8s.io/kustomize => sigs.k8s.io/kustomize v2.0.3+incompatible +<<<<<<< HEAD sigs.k8s.io/structured-merge-diff/v4 => sigs.k8s.io/structured-merge-diff/v4 v4.4.1 sigs.k8s.io/yaml => sigs.k8s.io/yaml v1.4.0 +======= + sigs.k8s.io/structured-merge-diff/v4 => sigs.k8s.io/structured-merge-diff/v4 v4.3.0 + sigs.k8s.io/yaml => sigs.k8s.io/yaml v1.3.0 +>>>>>>> feature/agent/new-agents-helm-template-base ) require ( @@ -377,24 +486,45 @@ require ( github.com/zeebo/xxh3 v1.0.2 go.etcd.io/bbolt v1.3.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 +<<<<<<< HEAD go.opentelemetry.io/otel v1.19.0 +======= + go.opentelemetry.io/otel v1.16.0 +>>>>>>> feature/agent/new-agents-helm-template-base go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.0.0-00010101000000-000000000000 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 +<<<<<<< HEAD go.opentelemetry.io/otel/metric v1.19.0 go.opentelemetry.io/otel/sdk v1.19.0 go.opentelemetry.io/otel/sdk/metric v0.33.0 go.opentelemetry.io/otel/trace v1.19.0 +======= + go.opentelemetry.io/otel/metric v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/sdk/metric v0.39.0 + go.opentelemetry.io/otel/trace v1.16.0 +>>>>>>> feature/agent/new-agents-helm-template-base go.uber.org/automaxprocs v0.0.0-00010101000000-000000000000 go.uber.org/goleak v1.2.1 go.uber.org/zap v1.26.0 gocloud.dev v0.0.0-00010101000000-000000000000 +<<<<<<< HEAD golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.15.0 golang.org/x/sync v0.5.0 golang.org/x/sys v0.15.0 golang.org/x/text v0.14.0 golang.org/x/tools v0.16.0 +======= + golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea + golang.org/x/net v0.12.0 + golang.org/x/oauth2 v0.10.0 + golang.org/x/sync v0.3.0 + golang.org/x/sys v0.10.0 + golang.org/x/text v0.11.0 + golang.org/x/tools v0.10.0 +>>>>>>> feature/agent/new-agents-helm-template-base gonum.org/v1/hdf5 v0.0.0-00010101000000-000000000000 gonum.org/v1/plot v0.10.1 google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f @@ -421,7 +551,7 @@ require ( github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/campoy/embedmd v1.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -434,9 +564,15 @@ require ( github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect +<<<<<<< HEAD github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect github.com/go-pdf/fpdf v0.9.0 // indirect +======= + github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-pdf/fpdf v0.8.0 // indirect +>>>>>>> feature/agent/new-agents-helm-template-base github.com/go-toolsmith/astcopy v1.0.2 // indirect github.com/go-toolsmith/astequal v1.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -452,7 +588,11 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.5.0 // indirect +<<<<<<< HEAD github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect +======= + github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect +>>>>>>> feature/agent/new-agents-helm-template-base github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect @@ -487,8 +627,8 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index cbeeccd22e4..43a440a37c8 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +<<<<<<< HEAD buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231115204500-e097f827e652.2 h1:iEPA5SBtdLJNwQis/SrcCuDWJh5E1V0mVO4Ih7/mRbg= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231115204500-e097f827e652.2/go.mod h1:xafc+XIsTxTy76GJQ1TKgvJWsSugFBqMaN27WhUblew= cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= @@ -29,6 +30,37 @@ cloud.google.com/go/clouddms v1.7.3/go.mod h1:fkN2HQQNUYInAU3NQ3vRLkV2iWs8lIdmBK cloud.google.com/go/cloudtasks v1.12.4/go.mod h1:BEPu0Gtt2dU6FxZHNqqNdGqIG86qyWKBPGnsb7udGY0= cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +======= +cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= +cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= +cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= +cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +>>>>>>> feature/agent/new-agents-helm-template-base cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= @@ -155,6 +187,30 @@ git.sr.ht/~sbinet/cmpimg v0.1.0/go.mod h1:FU12psLbF4TfNXkKH2ZZQ29crIqoiqTZmeQ7dk git.sr.ht/~sbinet/gg v0.5.0 h1:6V43j30HM623V329xA9Ntq+WJrMjDxRjuAB1LFWF5m8= git.sr.ht/~sbinet/gg v0.5.0/go.mod h1:G2C0eRESqlKhS7ErsNey6HHrqU1PwsnCQlekFi9Q2Oo= git.wow.st/gmp/jni v0.0.0-20210610011705-34026c7e22d0/go.mod h1:+axXBRUTIDlCeE73IKeD/os7LoEnTKdkp8/gQOFjqyo= +<<<<<<< HEAD +======= +github.com/Azure/azure-amqp-common-go/v3 v3.2.3/go.mod h1:7rPmbSfszeovxGfc5fSAXE4ehlXQZHpMja2OtxC2Tas= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.4.0/go.mod h1:pXDkeh10bAqElvd+S5Ppncj+DCKvJGXNa8rRT2R7rIw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/go-amqp v1.0.1/go.mod h1:+bg0x3ce5+Q3ahCEXnCsGG3ETpDQe3MEVnOuT2ywPwc= +github.com/Azure/go-autorest v14.2.1-0.20230628134501-d3f8f8a6cdf2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.30-0.20230628134501-d3f8f8a6cdf2/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/date v0.3.1-0.20230628134501-d3f8f8a6cdf2/go.mod h1:CHJ5Cu2H2HOHbo8wSOidbGEk/L11sCIy/IglS16YuRI= +github.com/Azure/go-autorest/autorest/mocks v0.4.3-0.20230628134501-d3f8f8a6cdf2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.1-0.20230628134501-d3f8f8a6cdf2/go.mod h1:DSh3VxN6gfEglfL5RXwMuwLQrHEayq9JF/NUe+DkxCI= +github.com/Azure/go-autorest/logger v0.2.2-0.20230628134501-d3f8f8a6cdf2/go.mod h1:mwDiRID/uD2cdq/qPxuXAZnmOEnn6qSn/xnC+0Q//eE= +github.com/Azure/go-autorest/tracing v0.6.1-0.20230628134501-d3f8f8a6cdf2/go.mod h1:DWJLY5/ttkqsBNafOmCvqIlik+Da9EFPSpzNnCWRjMo= +github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= +>>>>>>> feature/agent/new-agents-helm-template-base github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/DATA-DOG/go-sqlmock v1.5.1 h1:FK6RCIUSfmbnI/imIICmboyQBkOckutaa6R5YYlLZyo= github.com/DATA-DOG/go-sqlmock v1.5.1/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= @@ -164,8 +220,14 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +<<<<<<< HEAD github.com/ajstarks/deck v0.0.0-20231012031509-f833e437b68a/go.mod h1:CsIRRFEUjsvutRrtCL20i8EdMLo6JPw/F2XUmF+X694= github.com/ajstarks/deck/generate v0.0.0-20231012031509-f833e437b68a/go.mod h1:u04DhpZIpzaPnAUmhhjibCj450/2ITtLTEnvhaM49as= +======= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/ajstarks/deck v0.0.0-20230713212537-3862a8c503c2/go.mod h1:Nd40ZJeNA/I7Wu/RloLtis/eRLHLR/Q9YhYrDZndIeg= +github.com/ajstarks/deck/generate v0.0.0-20230713212537-3862a8c503c2/go.mod h1:u04DhpZIpzaPnAUmhhjibCj450/2ITtLTEnvhaM49as= +>>>>>>> feature/agent/new-agents-helm-template-base github.com/ajstarks/fc v0.0.0-20230606144319-ef5d5cb73a3d/go.mod h1:Qp3TfzbBiIjHwDxIpu+g9nYfNw+xXF2Yqp4WmMlTtwM= github.com/ajstarks/openvg v0.0.0-20191008131700-c6885d824eb8/go.mod h1:jpZHIkd4sQEgrzshrUQrRfv5OUMMq0w/Q1yK6ZYhUlk= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw= @@ -178,6 +240,7 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +<<<<<<< HEAD github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.49.0 h1:g9BkW1fo9GqKfwg2+zCD+TW/D36Ux+vtfJ8guF4AYmY= github.com/aws/aws-sdk-go v1.49.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= @@ -219,6 +282,55 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNIC github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +======= +github.com/aws/aws-sdk-go v1.44.300 h1:Zn+3lqgYahIf9yfrwZ+g+hq/c3KzUBaQ8wqY/ZXiAbY= +github.com/aws/aws-sdk-go v1.44.300/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= +github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= +github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27/go.mod h1:syOqAek45ZXZp29HlnRS/BNgMIW6uiRmeuQsz4Qh2UE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5/go.mod h1:Gj7tm95r+QsDoN2Fhuz/3npQvcZbkEf5mL70n3Xfluc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 h1:m0MmP89v1B0t3b8W8rtATU76KNsodak69QtiokHyEvo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72/go.mod h1:ylOTxIuoTL+XjH46Omv2iPjHdeGUk3SQ4hxYho4EHMA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 h1:cZG7psLfqpkB6H+fIrgUDWmlzM474St1LP0jcz272yI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27/go.mod h1:ZdjYvJpDlefgh8/hWelJhqgqJeodxu4SmbVsSdBlL7E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 h1:Bje8Xkh2OWpjBdNfXLrnn8eZg569dUQmhgtydxAYyP0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30/go.mod h1:qQtIBl5OVMfmeQkz8HaVyh5DzFmmFXyvK27UgIgOr4c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 h1:hx4WksB0NRQ9utR+2c3gEGzl6uKj3eM6PMQ6tN3lgXs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4/go.mod h1:JniVpqvw90sVjNqanGLufrVapWySL28fhBlYgl96Q/w= +github.com/aws/aws-sdk-go-v2/service/kms v1.23.1/go.mod h1:BuDl6WtqaDJbd9c29q/EFHrZjuWlrJN7oMNy5Yd5n7Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 h1:PalLOEGZ/4XfQxpGZFTLaoJSmPoybnqJYotaIZEf/Rg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0/go.mod h1:PwyKKVL0cNkC37QwLcrhyeCrAk+5bY8O2ou7USyAS2A= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.11/go.mod h1:ywwMMBG8ioLfw1LccY05t/egwONbtwDkoUVuXNgDZjs= +github.com/aws/aws-sdk-go-v2/service/sns v1.20.14/go.mod h1:bhIXgiDmP5qREwaPNFMyflXgjtd8USRrISXYQygSDi0= +github.com/aws/aws-sdk-go-v2/service/sqs v1.23.3/go.mod h1:FcXJKz137Ousb8wFnHyYI/qjUB7nUUFqKZvWaBe7Fy0= +github.com/aws/aws-sdk-go-v2/service/ssm v1.36.8/go.mod h1:Uwh2QwiXNf2+WCU3z5K13HE6f2bLCu9WpioFRkWjUVk= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13/go.mod h1:BzqsVVFduubEmzrVtUFQQIQdFqvUItF8XUq2EnS8Wog= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2OZgH1ypC114H04U= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +>>>>>>> feature/agent/new-agents-helm-template-base github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= @@ -229,6 +341,7 @@ github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY= github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -252,8 +365,14 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dUg3/RlKGr4HI= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +<<<<<<< HEAD github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +======= +github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= +github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +>>>>>>> feature/agent/new-agents-helm-template-base github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= @@ -402,7 +521,11 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +<<<<<<< HEAD github.com/ianlancetaylor/demangle v0.0.0-20231023195312-e2daf7ba7156/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +======= +github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +>>>>>>> feature/agent/new-agents-helm-template-base github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -429,10 +552,17 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.6.3/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +<<<<<<< HEAD github.com/klauspost/compress v1.17.5-0.20231209164634-6bf960e5bd5d h1:UbHXx+E5gnUabmdPHcMr/JqJsf6zb1NLThctXovfPNQ= github.com/klauspost/compress v1.17.5-0.20231209164634-6bf960e5bd5d/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +======= +github.com/klauspost/compress v1.16.8-0.20230714153001-f71e4d343a35 h1:+eclY3ILY5+eB1Zg5C1F01fBraSFI8z/l4qKvdixvdI= +github.com/klauspost/compress v1.16.8-0.20230714153001-f71e4d343a35/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +>>>>>>> feature/agent/new-agents-helm-template-base github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84= github.com/kpango/fastime v1.1.9/go.mod h1:vyD7FnUn08zxY4b/QFBZVG+9EWMYsNl+QF0uE46urD4= github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 h1:rxyM+7uaZQ35P9fbixdnld/h4AgEhODoubuy6A4nDdk= @@ -578,6 +708,7 @@ go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +<<<<<<< HEAD go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4= @@ -625,6 +756,58 @@ golang.org/x/exp/typeparams v0.0.0-20231206192017-f3f8817b8deb h1:O9ulz4QbYejvlk golang.org/x/exp/typeparams v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +======= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 h1:f6BwB2OACc3FCbYVznctQ9V6KK7Vq6CjmYXJ7DeSs4E= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0/go.mod h1:UqL5mZ3qs6XYhDnZaW1Ps4upD+PX6LipH40AoeuIlwU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 h1:rm+Fizi7lTM2UefJ1TO347fSRcwmIsUAaZmYmIGBRAo= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0/go.mod h1:sWFbI3jJ+6JdjOVepA5blpv/TJ20Hw+26561iMbWcwU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI= +go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +go.starlark.net v0.0.0-20230712173630-2226322290fc h1:x7dWtxLF8z8E5/+KkK3MJJTK/kBZhTCLmYCk75rhKxk= +go.starlark.net v0.0.0-20230712173630-2226322290fc/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= +go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +gocloud.dev v0.30.0 h1:PRgA+DXUz8/uuTJDA7wc8o2Hwj9yZ2qAsShZ60esbE8= +gocloud.dev v0.30.0/go.mod h1:w+GlGVg/Jy9JV0Xc9eSXzUZeVEmSWW49W0syFK1+T9U= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp/shiny v0.0.0-20220827204233-334a2380cb91/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8= +golang.org/x/exp/shiny v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= +golang.org/x/exp/typeparams v0.0.0-20230713183714-613f0c0eb8a1 h1:VXDua8UTGWl3e7L5kCk5Vyt0LA3QpsyRu6XXL7K3v1w= +golang.org/x/exp/typeparams v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/image v0.9.0 h1:QrzfX26snvCM20hIhBwuHI/ThTg18b/+kcKdXHvnR+g= +golang.org/x/image v0.9.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= +>>>>>>> feature/agent/new-agents-helm-template-base golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a/go.mod h1:Ede7gF0KGoHlj822RtphAHK1jLdrcuRBZg0sF1Q+SPc= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= @@ -653,6 +836,7 @@ gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0= gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= +<<<<<<< HEAD gonum.org/v1/plot v0.14.0 h1:+LBDVFYwFe4LHhdP8coW6296MBEY4nQ+Y4vuUpJopcE= gonum.org/v1/plot v0.14.0/go.mod h1:MLdR9424SJed+5VqC6MsouEpig9pZX2VZ57H9ko2bXU= google.golang.org/api v0.153.0 h1:N1AwGhielyKFaUqH07/ZSIQR3uNPcV7NVw0vj+j4iR4= @@ -668,6 +852,23 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20231211222908-989df2bf70f3 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20231211222908-989df2bf70f3/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +======= +gonum.org/v1/plot v0.13.0 h1:yb2Z/b8bY5h/xC4uix+ujJ+ixvPUvBmUOtM73CJzpsw= +gonum.org/v1/plot v0.13.0/go.mod h1:mV4Bpu4PWTgN2CETURNF8hCMg7EtlZqJYCcmYo/t4Co= +google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M= +google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:ZevU1kiy5jwy/sOVakfUuu3kq2Fwdgt3pORDQ2Jhkec= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +>>>>>>> feature/agent/new-agents-helm-template-base google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= @@ -759,7 +960,14 @@ sigs.k8s.io/kustomize/api v0.13.2 h1:kejWfLeJhUsTGioDoFNJET5LQe/ajzXhJGYoU+pJsiA sigs.k8s.io/kustomize/api v0.13.2/go.mod h1:DUp325VVMFVcQSq+ZxyDisA8wtldwHxLZbr1g94UHsw= sigs.k8s.io/kustomize/kyaml v0.14.1 h1:c8iibius7l24G2wVAGZn/Va2wNys03GXLjYVIcFVxKA= sigs.k8s.io/kustomize/kyaml v0.14.1/go.mod h1:AN1/IpawKilWD7V+YvQwRGUvuUOOWpjsHu6uHwonSF4= +<<<<<<< HEAD sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +======= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +>>>>>>> feature/agent/new-agents-helm-template-base diff --git a/hack/go.mod.default b/hack/go.mod.default index ba685da2ca4..d6e83bdc581 100755 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -191,6 +191,7 @@ replace ( github.com/kisielk/errcheck => github.com/kisielk/errcheck upgrade github.com/kisielk/gotool => github.com/kisielk/gotool upgrade github.com/klauspost/compress => github.com/klauspost/compress master +<<<<<<< HEAD github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 upgrade github.com/kpango/fastime => github.com/kpango/fastime upgrade github.com/kpango/fuid => github.com/kpango/fuid upgrade @@ -327,6 +328,142 @@ replace ( gopkg.in/yaml.v2 => gopkg.in/yaml.v2 upgrade gopkg.in/yaml.v3 => gopkg.in/yaml.v3 upgrade honnef.co/go/tools => honnef.co/go/tools upgrade +======= + github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 latest + github.com/kpango/fastime => github.com/kpango/fastime latest + github.com/kpango/fuid => github.com/kpango/fuid latest + github.com/kpango/gache/v2 => github.com/kpango/gache/v2 latest + github.com/kpango/glg => github.com/kpango/glg latest + github.com/kr/fs => github.com/kr/fs latest + github.com/kr/pretty => github.com/kr/pretty latest + github.com/kr/text => github.com/kr/text latest + github.com/kylelemons/godebug => github.com/kylelemons/godebug latest + github.com/leanovate/gopter => github.com/leanovate/gopter latest + github.com/leodido/go-urn => github.com/leodido/go-urn latest + github.com/lib/pq => github.com/lib/pq latest + github.com/liggitt/tabwriter => github.com/liggitt/tabwriter latest + github.com/lucasb-eyer/go-colorful => github.com/lucasb-eyer/go-colorful latest + github.com/mailru/easyjson => github.com/mailru/easyjson latest + github.com/mattn/go-colorable => github.com/mattn/go-colorable latest + github.com/mattn/go-isatty => github.com/mattn/go-isatty latest + github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 latest + github.com/matttproud/golang_protobuf_extensions => github.com/matttproud/golang_protobuf_extensions latest + github.com/mitchellh/colorstring => github.com/mitchellh/colorstring latest + github.com/moby/spdystream => github.com/moby/spdystream latest + github.com/moby/sys/mountinfo => github.com/moby/sys/mountinfo latest + github.com/modern-go/concurrent => github.com/modern-go/concurrent latest + github.com/modern-go/reflect2 => github.com/modern-go/reflect2 latest + github.com/modocache/gover => github.com/modocache/gover latest + github.com/monochromegane/go-gitignore => github.com/monochromegane/go-gitignore latest + github.com/montanaflynn/stats => github.com/montanaflynn/stats latest + github.com/munnerz/goautoneg => github.com/munnerz/goautoneg latest + github.com/niemeyer/pretty => github.com/niemeyer/pretty latest + github.com/nxadm/tail => github.com/nxadm/tail latest + github.com/onsi/ginkgo => github.com/onsi/ginkgo latest + github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 latest + github.com/onsi/gomega => github.com/onsi/gomega latest + github.com/peterbourgon/diskv => github.com/peterbourgon/diskv latest + github.com/phpdave11/gofpdf => github.com/phpdave11/gofpdf latest + github.com/phpdave11/gofpdi => github.com/phpdave11/gofpdi latest + github.com/pierrec/cmdflag => github.com/pierrec/cmdflag latest + github.com/pierrec/lz4/v3 => github.com/pierrec/lz4/v3 latest + github.com/pkg/browser => github.com/pkg/browser latest + github.com/pkg/errors => github.com/pkg/errors latest + github.com/pkg/sftp => github.com/pkg/sftp latest + github.com/pmezard/go-difflib => github.com/pmezard/go-difflib latest + github.com/prashantv/gostub => github.com/prashantv/gostub latest + github.com/prometheus/client_golang => github.com/prometheus/client_golang latest + github.com/prometheus/client_model => github.com/prometheus/client_model latest + github.com/prometheus/common => github.com/prometheus/common latest + github.com/prometheus/procfs => github.com/prometheus/procfs latest + github.com/prometheus/prometheus => github.com/prometheus/prometheus latest + github.com/quasilyte/go-ruleguard => github.com/quasilyte/go-ruleguard latest + github.com/quasilyte/go-ruleguard/dsl => github.com/quasilyte/go-ruleguard/dsl latest + github.com/quasilyte/gogrep => github.com/quasilyte/gogrep latest + github.com/quasilyte/stdinfo => github.com/quasilyte/stdinfo latest + github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid latest + github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal latest + github.com/rs/xid => github.com/rs/xid latest + github.com/rs/zerolog => github.com/rs/zerolog latest + github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 latest + github.com/ruudk/golang-pdf417 => github.com/ruudk/golang-pdf417 latest + github.com/schollz/progressbar/v2 => github.com/schollz/progressbar/v2 latest + github.com/scylladb/go-reflectx => github.com/scylladb/go-reflectx latest + github.com/scylladb/gocqlx => github.com/scylladb/gocqlx latest + github.com/sergi/go-diff => github.com/sergi/go-diff latest + github.com/shopspring/decimal => github.com/shopspring/decimal latest + github.com/shurcooL/httpfs => github.com/shurcooL/httpfs latest + github.com/shurcooL/vfsgen => github.com/shurcooL/vfsgen latest + github.com/sirupsen/logrus => github.com/sirupsen/logrus latest + github.com/spf13/afero => github.com/spf13/afero latest + github.com/spf13/cobra => github.com/spf13/cobra latest + github.com/spf13/pflag => github.com/spf13/pflag latest + github.com/stoewer/go-strcase => github.com/stoewer/go-strcase latest + github.com/stretchr/objx => github.com/stretchr/objx latest + github.com/stretchr/testify => github.com/stretchr/testify latest + github.com/ugorji/go/codec => github.com/ugorji/go/codec latest + github.com/xeipuuv/gojsonpointer => github.com/xeipuuv/gojsonpointer latest + github.com/xeipuuv/gojsonreference => github.com/xeipuuv/gojsonreference latest + github.com/xeipuuv/gojsonschema => github.com/xeipuuv/gojsonschema latest + github.com/xlab/treeprint => github.com/xlab/treeprint latest + github.com/zeebo/assert => github.com/zeebo/assert latest + github.com/zeebo/xxh3 => github.com/zeebo/xxh3 latest + go.opencensus.io => go.opencensus.io latest + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc latest + go.opentelemetry.io/otel => go.opentelemetry.io/otel latest + go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry latest + go.opentelemetry.io/otel/exporters/otlp/otlpmetric => go.opentelemetry.io/otel/exporters/otlp/otlpmetric latest + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc latest + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace latest + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc latest + go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric latest + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk latest + go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric latest + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace latest + go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp latest + go.starlark.net => go.starlark.net latest + go.uber.org/atomic => go.uber.org/atomic latest + go.uber.org/automaxprocs => go.uber.org/automaxprocs latest + go.uber.org/goleak => go.uber.org/goleak latest + go.uber.org/multierr => go.uber.org/multierr latest + go.uber.org/zap => go.uber.org/zap latest + gocloud.dev => gocloud.dev latest + golang.org/x/crypto => golang.org/x/crypto latest + golang.org/x/exp => golang.org/x/exp latest + golang.org/x/exp/typeparams => golang.org/x/exp/typeparams latest + golang.org/x/image => golang.org/x/image latest + golang.org/x/lint => golang.org/x/lint latest + golang.org/x/mobile => golang.org/x/mobile latest + golang.org/x/mod => golang.org/x/mod latest + golang.org/x/net => golang.org/x/net latest + golang.org/x/oauth2 => golang.org/x/oauth2 latest + golang.org/x/sync => golang.org/x/sync latest + golang.org/x/sys => golang.org/x/sys latest + golang.org/x/term => golang.org/x/term latest + golang.org/x/text => golang.org/x/text latest + golang.org/x/time => golang.org/x/time latest + golang.org/x/tools => golang.org/x/tools latest + golang.org/x/xerrors => golang.org/x/xerrors latest + gomodules.xyz/jsonpatch/v2 => gomodules.xyz/jsonpatch/v2 latest + gonum.org/v1/gonum => gonum.org/v1/gonum latest + gonum.org/v1/hdf5 => gonum.org/v1/hdf5 latest + gonum.org/v1/plot => gonum.org/v1/plot latest + google.golang.org/api => google.golang.org/api latest + google.golang.org/appengine => google.golang.org/appengine latest + google.golang.org/genproto => google.golang.org/genproto latest + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api latest + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc latest + google.golang.org/grpc => google.golang.org/grpc latest + google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc latest + google.golang.org/protobuf => google.golang.org/protobuf latest + gopkg.in/check.v1 => gopkg.in/check.v1 latest + gopkg.in/inconshreveable/log15.v2 => gopkg.in/inconshreveable/log15.v2 latest + gopkg.in/inf.v0 => gopkg.in/inf.v0 latest + gopkg.in/tomb.v1 => gopkg.in/tomb.v1 latest + gopkg.in/yaml.v2 => gopkg.in/yaml.v2 latest + gopkg.in/yaml.v3 => gopkg.in/yaml.v3 latest + honnef.co/go/tools => honnef.co/go/tools latest +>>>>>>> feature/agent/new-agents-helm-template-base k8s.io/api => k8s.io/api v0.27.3 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.27.3 k8s.io/apimachinery => k8s.io/apimachinery v0.27.3 diff --git a/hack/license/gen/main.go b/hack/license/gen/main.go index 9ce87051809..17e38d0900a 100644 --- a/hack/license/gen/main.go +++ b/hack/license/gen/main.go @@ -209,6 +209,7 @@ func dirwalk(dir string) []string { "CONTRIBUTORS", "GO_VERSION", "NGT_VERSION", + "FAISS_VERSION", "Pipefile", "VALD_VERSION", "grp", @@ -259,7 +260,11 @@ func readAndRewrite(path string) error { } else { tmpl := apache switch filepath.Ext(path) { +<<<<<<< HEAD case ".go": +======= + case ".go", ".proto", ".c", ".h", ".hpp", ".cpp": +>>>>>>> feature/agent/new-agents-helm-template-base d.Escape = slushEscape switch fi.Name() { case "errgroup_test.go", diff --git a/internal/client/v1/client/agent/core/client.go b/internal/client/v1/client/agent/core/client.go index bbf17abee48..82528e86a27 100644 --- a/internal/client/v1/client/agent/core/client.go +++ b/internal/client/v1/client/agent/core/client.go @@ -86,11 +86,7 @@ func New(opts ...Option) (Client, error) { return c, nil } -func NewAgentClient(cc *grpc.ClientConn) interface { - vald.Client - client.ObjectReader - client.Indexer -} { +func NewAgentClient(cc *grpc.ClientConn) Client { return &singleAgentClient{ Client: vald.NewValdClient(cc), ac: agent.NewAgentClient(cc), diff --git a/internal/config/faiss.go b/internal/config/faiss.go new file mode 100644 index 00000000000..732d5a26be1 --- /dev/null +++ b/internal/config/faiss.go @@ -0,0 +1,118 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package config providers configuration type and load configuration logic +package config + +// Faiss represent the faiss core configuration for server. +type Faiss struct { + // IndexPath represents the faiss index file path + IndexPath string `yaml:"index_path" json:"index_path,omitempty"` + + // Dimension represents the faiss index dimension + Dimension int `yaml:"dimension" json:"dimension,omitempty" info:"dimension"` + + // Nlist represents the number of Voronoi cells + // ref: https://github.com/facebookresearch/faiss/wiki/Faster-search + Nlist int `yaml:"nlist" json:"nlist,omitempty" info:"nlist"` + + // M represents the number of subquantizers + // ref: https://github.com/facebookresearch/faiss/wiki/Faiss-indexes-(composite)#cell-probe-method-with-a-pq-index-as-coarse-quantizer + M int `yaml:"m" json:"m,omitempty" info:"m"` + + // NbitsPerIdx represents the number of bit per subvector index + // ref: https://github.com/facebookresearch/faiss/wiki/FAQ#can-i-ignore-warning-clustering-xxx-points-to-yyy-centroids + NbitsPerIdx int `yaml:"nbits_per_idx" json:"nbits_per_idx,omitempty" info:"nbits_per_idx"` + + // MetricType represents the metric type + MetricType string `yaml:"metric_type" json:"metric_type,omitempty" info:"metric_type"` + + // EnableInMemoryMode enables on memory faiss indexing mode + EnableInMemoryMode bool `yaml:"enable_in_memory_mode" json:"enable_in_memory_mode,omitempty"` + + // AutoIndexCheckDuration represents checking loop duration about auto indexing execution + AutoIndexCheckDuration string `yaml:"auto_index_check_duration" json:"auto_index_check_duration,omitempty"` + + // AutoSaveIndexDuration represents checking loop duration about auto save index execution + AutoSaveIndexDuration string `yaml:"auto_save_index_duration" json:"auto_save_index_duration,omitempty"` + + // AutoIndexDurationLimit represents auto indexing duration limit + AutoIndexDurationLimit string `yaml:"auto_index_duration_limit" json:"auto_index_duration_limit,omitempty"` + + // AutoIndexLength represents auto index length limit + AutoIndexLength int `yaml:"auto_index_length" json:"auto_index_length,omitempty"` + + // InitialDelayMaxDuration represents maximum duration for initial delay + InitialDelayMaxDuration string `yaml:"initial_delay_max_duration" json:"initial_delay_max_duration,omitempty"` + + // MinLoadIndexTimeout represents minimum duration of load index timeout + MinLoadIndexTimeout string `yaml:"min_load_index_timeout" json:"min_load_index_timeout,omitempty"` + + // MaxLoadIndexTimeout represents maximum duration of load index timeout + MaxLoadIndexTimeout string `yaml:"max_load_index_timeout" json:"max_load_index_timeout,omitempty"` + + // LoadIndexTimeoutFactor represents a factor of load index timeout + LoadIndexTimeoutFactor string `yaml:"load_index_timeout_factor" json:"load_index_timeout_factor,omitempty"` + + // EnableProactiveGC enables more proactive GC call for reducing heap memory allocation + EnableProactiveGC bool `yaml:"enable_proactive_gc" json:"enable_proactive_gc,omitempty"` + + // EnableCopyOnWrite enables copy on write saving + EnableCopyOnWrite bool `yaml:"enable_copy_on_write" json:"enable_copy_on_write,omitempty"` + + // VQueue represents the faiss vector queue buffer size + VQueue *VQueue `json:"vqueue,omitempty" yaml:"vqueue"` + + // KVSDB represents the faiss bidirectional kv store configuration + KVSDB *KVSDB `json:"kvsdb,omitempty" yaml:"kvsdb"` +} + +//// KVSDB represent the faiss vector bidirectional kv store configuration +//type KVSDB struct { +// // Concurrency represents kvsdb range loop processing concurrency +// Concurrency int `json:"concurrency,omitempty" yaml:"concurrency,omitempty"` +//} +// +//// VQueue represent the faiss vector queue buffer size +//type VQueue struct { +// // InsertBufferPoolSize represents insert time ordered slice buffer size +// InsertBufferPoolSize int `json:"insert_buffer_pool_size,omitempty" yaml:"insert_buffer_pool_size"` +// +// // DeleteBufferPoolSize represents delete time ordered slice buffer size +// DeleteBufferPoolSize int `json:"delete_buffer_pool_size,omitempty" yaml:"delete_buffer_pool_size"` +//} + +// Bind returns Faiss object whose some string value is filed value or environment value. +func (f *Faiss) Bind() *Faiss { + f.IndexPath = GetActualValue(f.IndexPath) + f.MetricType = GetActualValue(f.MetricType) + f.AutoIndexCheckDuration = GetActualValue(f.AutoIndexCheckDuration) + f.AutoIndexDurationLimit = GetActualValue(f.AutoIndexDurationLimit) + f.AutoSaveIndexDuration = GetActualValue(f.AutoSaveIndexDuration) + f.InitialDelayMaxDuration = GetActualValue(f.InitialDelayMaxDuration) + f.MinLoadIndexTimeout = GetActualValue(f.MinLoadIndexTimeout) + f.MaxLoadIndexTimeout = GetActualValue(f.MaxLoadIndexTimeout) + f.LoadIndexTimeoutFactor = GetActualValue(f.LoadIndexTimeoutFactor) + + if f.VQueue == nil { + f.VQueue = new(VQueue) + } + if f.KVSDB == nil { + f.KVSDB = new(KVSDB) + } + + return f +} diff --git a/internal/config/faiss_test.go b/internal/config/faiss_test.go new file mode 100644 index 00000000000..dcecf87c444 --- /dev/null +++ b/internal/config/faiss_test.go @@ -0,0 +1,177 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestFaiss_Bind(t *testing.T) { +// type fields struct { +// IndexPath string +// Dimension int +// Nlist int +// M int +// NbitsPerIdx int +// MetricType string +// EnableInMemoryMode bool +// AutoIndexCheckDuration string +// AutoSaveIndexDuration string +// AutoIndexDurationLimit string +// AutoIndexLength int +// InitialDelayMaxDuration string +// MinLoadIndexTimeout string +// MaxLoadIndexTimeout string +// LoadIndexTimeoutFactor string +// EnableProactiveGC bool +// EnableCopyOnWrite bool +// VQueue *VQueue +// KVSDB *KVSDB +// } +// type want struct { +// want *Faiss +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, *Faiss) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got *Faiss) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// IndexPath:"", +// Dimension:0, +// Nlist:0, +// M:0, +// NbitsPerIdx:0, +// MetricType:"", +// EnableInMemoryMode:false, +// AutoIndexCheckDuration:"", +// AutoSaveIndexDuration:"", +// AutoIndexDurationLimit:"", +// AutoIndexLength:0, +// InitialDelayMaxDuration:"", +// MinLoadIndexTimeout:"", +// MaxLoadIndexTimeout:"", +// LoadIndexTimeoutFactor:"", +// EnableProactiveGC:false, +// EnableCopyOnWrite:false, +// VQueue:VQueue{}, +// KVSDB:KVSDB{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// IndexPath:"", +// Dimension:0, +// Nlist:0, +// M:0, +// NbitsPerIdx:0, +// MetricType:"", +// EnableInMemoryMode:false, +// AutoIndexCheckDuration:"", +// AutoSaveIndexDuration:"", +// AutoIndexDurationLimit:"", +// AutoIndexLength:0, +// InitialDelayMaxDuration:"", +// MinLoadIndexTimeout:"", +// MaxLoadIndexTimeout:"", +// LoadIndexTimeoutFactor:"", +// EnableProactiveGC:false, +// EnableCopyOnWrite:false, +// VQueue:VQueue{}, +// KVSDB:KVSDB{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &Faiss{ +// IndexPath: test.fields.IndexPath, +// Dimension: test.fields.Dimension, +// Nlist: test.fields.Nlist, +// M: test.fields.M, +// NbitsPerIdx: test.fields.NbitsPerIdx, +// MetricType: test.fields.MetricType, +// EnableInMemoryMode: test.fields.EnableInMemoryMode, +// AutoIndexCheckDuration: test.fields.AutoIndexCheckDuration, +// AutoSaveIndexDuration: test.fields.AutoSaveIndexDuration, +// AutoIndexDurationLimit: test.fields.AutoIndexDurationLimit, +// AutoIndexLength: test.fields.AutoIndexLength, +// InitialDelayMaxDuration: test.fields.InitialDelayMaxDuration, +// MinLoadIndexTimeout: test.fields.MinLoadIndexTimeout, +// MaxLoadIndexTimeout: test.fields.MaxLoadIndexTimeout, +// LoadIndexTimeoutFactor: test.fields.LoadIndexTimeoutFactor, +// EnableProactiveGC: test.fields.EnableProactiveGC, +// EnableCopyOnWrite: test.fields.EnableCopyOnWrite, +// VQueue: test.fields.VQueue, +// KVSDB: test.fields.KVSDB, +// } +// +// got := f.Bind() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/config/server.go b/internal/config/server.go index 5bbf8564ee9..2aa17ab58b4 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -20,6 +20,7 @@ package config import ( "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/admin" "github.com/vdaas/vald/internal/net/grpc/health" "github.com/vdaas/vald/internal/net/grpc/reflection" "github.com/vdaas/vald/internal/servers/server" @@ -89,6 +90,7 @@ type GRPC struct { HeaderTableSize int `json:"header_table_size,omitempty" yaml:"header_table_size"` Interceptors []string `json:"interceptors,omitempty" yaml:"interceptors"` EnableReflection bool `json:"enable_reflection,omitempty" yaml:"enable_reflection"` + EnableAdmin bool `json:"enable_admin,omitempty" yaml:"enable_admin"` } // GRPCKeepalive represents the configuration for gRPC keep-alive. @@ -282,6 +284,12 @@ func (s *Server) Opts() []server.Option { reflection.Register(srv) })) } + if s.GRPC.EnableAdmin { + opts = append(opts, + server.WithGRPCRegistFunc(func(srv *grpc.Server) { + admin.Register(srv) + })) + } if s.GRPC.Keepalive != nil { opts = append(opts, server.WithGRPCKeepaliveMaxConnIdle(s.GRPC.Keepalive.MaxConnIdle), diff --git a/internal/core/algorithm/faiss/Capi.cpp b/internal/core/algorithm/faiss/Capi.cpp new file mode 100644 index 00000000000..7f1df856037 --- /dev/null +++ b/internal/core/algorithm/faiss/Capi.cpp @@ -0,0 +1,286 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Capi.h" + +#if DEBUG +constexpr bool isDebug = true; +#else +constexpr bool isDebug = false; +#endif + +FaissStruct* faiss_create_index( + const int d, + const int nlist, + const int m, + const int nbits_per_idx, + const int metric_type) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + FaissStruct *st = NULL; + try { + faiss::IndexFlat *quantizer; + switch (metric_type) { + case faiss::METRIC_INNER_PRODUCT: + quantizer = new faiss::IndexFlat(d, faiss::METRIC_INNER_PRODUCT); + break; + case faiss::METRIC_L2: + quantizer = new faiss::IndexFlat(d, faiss::METRIC_L2); + break; + default: + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: no metric type."; + std::cerr << ss.str() << std::endl; + return NULL; + } + faiss::IndexIVFPQ *index = new faiss::IndexIVFPQ(quantizer, d, nlist, m, nbits_per_idx); + if (isDebug) { + index->verbose = true; + } + st = new FaissStruct{ + static_cast(quantizer), + static_cast(index) + }; + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + } + + return st; +} + +FaissStruct* faiss_read_index(const char* fname) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + FaissStruct *st = NULL; + try { + st = new FaissStruct{ + static_cast(NULL), + static_cast(faiss::read_index(fname)) + }; + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + } + + if (isDebug) { + fflush(stdout); + } + return st; +} + +bool faiss_write_index( + const FaissStruct* st, + const char* fname) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + try { + faiss::write_index(static_cast(st->faiss_index), fname); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return false; + } + + if (isDebug) { + fflush(stdout); + } + return true; +} + +bool faiss_train( + const FaissStruct* st, + const int nb, + const float* xb) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + try { + if (isDebug) { + printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } + (static_cast(st->faiss_index))->train(nb, xb); + if (isDebug) { + printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return false; + } + + if (isDebug) { + fflush(stdout); + } + return true; +} + +int faiss_add( + const FaissStruct* st, + const int nb, + const float* xb, + const long int* xids ) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + try { + if (isDebug) { + printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } + (static_cast(st->faiss_index))->add_with_ids(nb, xb, xids); + if (isDebug) { + printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return -1; + } + + if (isDebug) { + fflush(stdout); + } + return (static_cast(st->faiss_index))->ntotal; +} + +bool faiss_search( + const FaissStruct* st, + const int k, + const int nq, + const float* xq, + long* I, + float* D) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + try { + if (isDebug) { + printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } + (static_cast(st->faiss_index))->search(nq, xq, k, D, I); + if (isDebug) { + printf("I=\n"); + for(int i = 0; i < nq; i++) { + for(int j = 0; j < k; j++) { + printf("%5ld ", I[i * k + j]); + } + printf("\n"); + } + printf("D=\n"); + for(int i = 0; i < nq; i++) { + for(int j = 0; j < k; j++) { + printf("%7g ", D[i * k + j]); + } + printf("\n"); + } + } + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return false; + } + + if (isDebug) { + fflush(stdout); + } + return true; +} + +int faiss_remove( + const FaissStruct* st, + const int size, + const long int* ids) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + try { + if (isDebug) { + printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } + faiss::IDSelectorArray sel(size, ids); + (static_cast(st->faiss_index))->remove_ids(sel); + if (isDebug) { + printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return -1; + } + + if (isDebug) { + fflush(stdout); + } + return (static_cast(st->faiss_index))->ntotal; +} + +void faiss_free(FaissStruct* st) { + if (isDebug) { + printf(__FUNCTION__); + printf("\n"); + fflush(stdout); + } + + free(st); + return; +} diff --git a/internal/core/algorithm/faiss/Capi.h b/internal/core/algorithm/faiss/Capi.h new file mode 100644 index 00000000000..72796672c77 --- /dev/null +++ b/internal/core/algorithm/faiss/Capi.h @@ -0,0 +1,68 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef DEBUG +#define DEBUG +#endif + +#ifdef __cplusplus +extern "C" { +#endif + #include + #include + #include + + typedef void* FaissQuantizer; + typedef void* FaissIndex; + typedef struct { + FaissQuantizer faiss_quantizer; + FaissIndex faiss_index; + } FaissStruct; + + FaissStruct* faiss_create_index( + const int d, + const int nlist, + const int m, + const int nbits_per_idx, + const int metric_type); + FaissStruct* faiss_read_index(const char* fname); + bool faiss_write_index( + const FaissStruct* st, + const char* fname); + bool faiss_train( + const FaissStruct* st, + const int nb, + const float* xb); + int faiss_add( + const FaissStruct* st, + const int nb, + const float* xb, + const long int* xids); + bool faiss_search( + const FaissStruct* st, + const int k, + const int nq, + const float* xq, + long* I, + float* D); + int faiss_remove( + const FaissStruct* st, + const int size, + const long int* ids); + void faiss_free(FaissStruct* st); +#ifdef __cplusplus +} +#endif diff --git a/internal/core/algorithm/faiss/faiss.go b/internal/core/algorithm/faiss/faiss.go new file mode 100644 index 00000000000..b7a0e605186 --- /dev/null +++ b/internal/core/algorithm/faiss/faiss.go @@ -0,0 +1,255 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package faiss provides implementation of Go API for https://github.com/facebookresearch/faiss +package faiss + +/* +#cgo LDFLAGS: -lfaiss +#include +*/ +import "C" + +import ( + "sync" + "unsafe" + + "github.com/vdaas/vald/internal/errors" +) + +type ( + // Faiss is core interface. + Faiss interface { + // SaveIndex stores faiss index to strage. + SaveIndex() error + + // SaveIndexWithPath stores faiss index to specified storage. + SaveIndexWithPath(idxPath string) error + + // Train trains faiss index. + Train(nb int, xb []float32) error + + // Add returns faiss ntotal. + Add(nb int, xb []float32, xids []int64) (int, error) + + // Search returns search result as []SearchResult. + Search(k, nq int, xq []float32) ([]SearchResult, error) + + // Remove removes from faiss index. + Remove(size int, ids []int64) (int, error) + + // Close faiss index. + Close() + } + + faiss struct { + st *C.FaissStruct + dimension C.int + nlist C.int + m C.int + nbitsPerIdx C.int + metricType metricType + idxPath string + mu *sync.RWMutex + } + + SearchResult struct { + ID uint32 + Distance float32 + Error error + } +) + +// metricType is alias of metric type in Faiss. +type metricType int + +const ( + // ------------------------------------------------------------- + // Metric Type Definition + // (https://github.com/facebookresearch/faiss/wiki/MetricType-and-distances) + // ------------------------------------------------------------- + // DistanceNone is unknown distance type. + DistanceNone metricType = iota - 1 + // InnerProduct is inner product. + InnerProduct + // L2 is l2 norm. + L2 + // -------------------------------------------------------------. + + // ------------------------------------------------------------- + // ErrorCode is false + // -------------------------------------------------------------. + ErrorCode = C._Bool(false) + // -------------------------------------------------------------. +) + +// New returns Faiss instance with recreating empty index file. +func New(opts ...Option) (Faiss, error) { + return gen(false, opts...) +} + +func Load(opts ...Option) (Faiss, error) { + return gen(true, opts...) +} + +func gen(isLoad bool, opts ...Option) (Faiss, error) { + var ( + f = new(faiss) + err error + ) + f.mu = new(sync.RWMutex) + + defer func() { + if err != nil { + f.Close() + } + }() + + for _, opt := range append(defaultOptions, opts...) { + if err = opt(f); err != nil { + return nil, errors.NewFaissError("faiss option error") + } + } + + if isLoad { + path := C.CString(f.idxPath) + defer C.free(unsafe.Pointer(path)) + f.st = C.faiss_read_index(path) + if f.st == nil { + return nil, errors.NewFaissError("faiss load index error") + } + } else { + switch f.metricType { + case InnerProduct: + f.st = C.faiss_create_index(f.dimension, f.nlist, f.m, f.nbitsPerIdx, C.int(InnerProduct)) + case L2: + f.st = C.faiss_create_index(f.dimension, f.nlist, f.m, f.nbitsPerIdx, C.int(L2)) + default: + return nil, errors.NewFaissError("faiss create index error: no metric type") + } + if f.st == nil { + return nil, errors.NewFaissError("faiss create index error: nil pointer") + } + } + + return f, nil +} + +// SaveIndex stores faiss index to storage. +func (f *faiss) SaveIndex() error { + path := C.CString(f.idxPath) + defer C.free(unsafe.Pointer(path)) + + f.mu.Lock() + ret := C.faiss_write_index(f.st, path) + f.mu.Unlock() + if ret == ErrorCode { + return errors.NewFaissError("failed to faiss_write_index") + } + + return nil +} + +// SaveIndexWithPath stores faiss index to specified storage. +func (f *faiss) SaveIndexWithPath(idxPath string) error { + path := C.CString(idxPath) + defer C.free(unsafe.Pointer(path)) + + f.mu.Lock() + ret := C.faiss_write_index(f.st, path) + f.mu.Unlock() + if ret == ErrorCode { + return errors.NewFaissError("failed to faiss_write_index") + } + + return nil +} + +// Train trains faiss index. +func (f *faiss) Train(nb int, xb []float32) error { + f.mu.Lock() + ret := C.faiss_train(f.st, (C.int)(nb), (*C.float)(&xb[0])) + f.mu.Unlock() + if ret == ErrorCode { + return errors.NewFaissError("failed to faiss_train") + } + + return nil +} + +// Add returns faiss ntotal. +func (f *faiss) Add(nb int, xb []float32, xids []int64) (int, error) { + dim := int(f.dimension) + if len(xb) != dim*nb || len(xb) != dim*len(xids) { + return -1, errors.ErrIncompatibleDimensionSize(len(xb)/nb, dim) + } + + f.mu.Lock() + ntotal := int(C.faiss_add(f.st, (C.int)(nb), (*C.float)(&xb[0]), (*C.long)(&xids[0]))) + f.mu.Unlock() + if ntotal < 0 { + return ntotal, errors.NewFaissError("failed to faiss_add") + } + + return ntotal, nil +} + +// Search returns search result as []SearchResult. +func (f *faiss) Search(k, nq int, xq []float32) ([]SearchResult, error) { + if len(xq) != nq*int(f.dimension) { + return nil, errors.ErrIncompatibleDimensionSize(len(xq), int(f.dimension)) + } + + I := make([]int64, k*nq) + D := make([]float32, k*nq) + f.mu.RLock() + ret := C.faiss_search(f.st, (C.int)(k), (C.int)(nq), (*C.float)(&xq[0]), (*C.long)(&I[0]), (*C.float)(&D[0])) + f.mu.RUnlock() + if ret == ErrorCode { + return nil, errors.NewFaissError("failed to faiss_search") + } + + if len(I) == 0 || len(D) == 0 { + return nil, errors.ErrEmptySearchResult + } + + result := make([]SearchResult, k) + for i := range result { + result[i] = SearchResult{uint32(I[i]), D[i], nil} + } + + return result, nil +} + +// Remove removes from faiss index. +func (f *faiss) Remove(size int, ids []int64) (int, error) { + f.mu.Lock() + ntotal := int(C.faiss_remove(f.st, (C.int)(size), (*C.long)(&ids[0]))) + f.mu.Unlock() + if ntotal < 0 { + return ntotal, errors.NewFaissError("failed to faiss_remove") + } + + return ntotal, nil +} + +// Close faiss index. +func (f *faiss) Close() { + if f.st != nil { + C.faiss_free(f.st) + f.st = nil + } +} diff --git a/internal/core/algorithm/faiss/option.go b/internal/core/algorithm/faiss/option.go new file mode 100644 index 00000000000..40c9a3ad5a0 --- /dev/null +++ b/internal/core/algorithm/faiss/option.go @@ -0,0 +1,120 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package faiss provides implementation of Go API for https://github.com/facebookresearch/faiss +package faiss + +import "C" + +import ( + "strings" + + "github.com/vdaas/vald/internal/core/algorithm" + "github.com/vdaas/vald/internal/errors" +) + +// Option represents the functional option for faiss. +type Option func(*faiss) error + +var defaultOptions = []Option{ + WithDimension(64), + WithNlist(100), + WithM(8), + WithNbitsPerIdx(8), + WithMetricType("l2"), +} + +// WithDimension represents the option to set the dimension for faiss. +func WithDimension(dim int) Option { + return func(f *faiss) error { + if dim > algorithm.MaximumVectorDimensionSize || dim < algorithm.MinimumVectorDimensionSize { + err := errors.ErrInvalidDimensionSize(dim, algorithm.MaximumVectorDimensionSize) + return errors.NewErrCriticalOption("dimension", dim, err) + } + + f.dimension = (C.int)(dim) + return nil + } +} + +// WithNlist represents the option to set the nlist for faiss. +func WithNlist(nlist int) Option { + return func(f *faiss) error { + if nlist <= 0 { + return errors.NewErrInvalidOption("nlist", nlist) + } + + f.nlist = (C.int)(nlist) + return nil + } +} + +// WithM represents the option to set the m for faiss. +func WithM(m int) Option { + return func(f *faiss) error { + if m <= 0 || int(f.dimension)%m != 0 { + return errors.NewErrInvalidOption("m", m) + } + + f.m = (C.int)(m) + return nil + } +} + +// WithNbitsPerIdx represents the option to set the n bits per index for faiss. +func WithNbitsPerIdx(nbitsPerIdx int) Option { + return func(f *faiss) error { + if nbitsPerIdx <= 0 { + return errors.NewErrInvalidOption("nbitsPerIdx", nbitsPerIdx) + } + + f.nbitsPerIdx = (C.int)(nbitsPerIdx) + return nil + } +} + +// WithMetricType represents the option to set the metric type for faiss. +func WithMetricType(metricType string) Option { + return func(f *faiss) error { + if len(metricType) == 0 { + return errors.NewErrIgnoredOption("metricType") + } + + switch strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(metricType)) { + case "innerproduct": + f.metricType = InnerProduct + case "l2": + f.metricType = L2 + default: + err := errors.ErrUnsupportedDistanceType + return errors.NewErrCriticalOption("metricType", metricType, err) + } + + return nil + } +} + +// WithIndexPath represents the option to set the index path for faiss. +func WithIndexPath(idxPath string) Option { + return func(f *faiss) error { + if len(idxPath) == 0 { + return errors.NewErrIgnoredOption("indexPath") + } + + f.idxPath = idxPath + return nil + } +} diff --git a/internal/errors/faiss.go b/internal/errors/faiss.go new file mode 100644 index 00000000000..735942079c9 --- /dev/null +++ b/internal/errors/faiss.go @@ -0,0 +1,32 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package errors provides error types and function +package errors + +type FaissError struct { + Msg string +} + +func NewFaissError(msg string) error { + return FaissError{ + Msg: msg, + } +} + +func (f FaissError) Error() string { + return f.Msg +} diff --git a/internal/errors/faiss_test.go b/internal/errors/faiss_test.go new file mode 100644 index 00000000000..175e1932ee0 --- /dev/null +++ b/internal/errors/faiss_test.go @@ -0,0 +1,191 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package errors + +// NOT IMPLEMENTED BELOW +// +// func TestNewFaissError(t *testing.T) { +// type args struct { +// msg string +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !Is(err, w.err) { +// return Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// msg:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// msg:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// err := NewFaissError(test.args.msg) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestFaissError_Error(t *testing.T) { +// type fields struct { +// Msg string +// } +// type want struct { +// want string +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, string) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got string) error { +// if !reflect.DeepEqual(got, w.want) { +// return Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// Msg:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// Msg:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := FaissError{ +// Msg: test.fields.Msg, +// } +// +// got := f.Error() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/info/info.go b/internal/info/info.go index c375145f501..0f022deb9ef 100644 --- a/internal/info/info.go +++ b/internal/info/info.go @@ -57,7 +57,7 @@ type Detail struct { GoArch string `json:"go_arch,omitempty" yaml:"go_arch,omitempty"` GoRoot string `json:"go_root,omitempty" yaml:"go_root,omitempty"` CGOEnabled string `json:"cgo_enabled,omitempty" yaml:"cgo_enabled,omitempty"` - NGTVersion string `json:"ngt_version,omitempty" yaml:"ngt_version,omitempty"` + AlgorithmInfo string `json:"algorithm_info,omitempty" yaml:"algorithm_info,omitempty"` BuildCPUInfoFlags []string `json:"build_cpu_info_flags,omitempty" yaml:"build_cpu_info_flags,omitempty"` StackTrace []StackTrace `json:"stack_trace,omitempty" yaml:"stack_trace,omitempty"` } @@ -89,8 +89,8 @@ var ( GoRoot string // CGOEnabled represent the cgo is enable or not to build Vald. CGOEnabled string - // NGTVersion represent the NGT version in Vald. - NGTVersion string + // AlgorithmInfo represent the NGT version in Vald. + AlgorithmInfo string // BuildCPUInfoFlags represent the CPU info flags to build Vald. BuildCPUInfoFlags string @@ -146,7 +146,7 @@ func New(opts ...Option) (Info, error) { GoArch: GoArch, GoRoot: GoRoot, CGOEnabled: CGOEnabled, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -361,8 +361,8 @@ func (i *info) prepare() { default: i.detail.CGOEnabled = cgoUnknown } - if len(i.detail.NGTVersion) == 0 && len(NGTVersion) != 0 { - i.detail.NGTVersion = NGTVersion + if len(i.detail.AlgorithmInfo) == 0 && len(AlgorithmInfo) != 0 { + i.detail.AlgorithmInfo = AlgorithmInfo } if len(i.detail.BuildCPUInfoFlags) == 0 && len(BuildCPUInfoFlags) != 0 { i.detail.BuildCPUInfoFlags = strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " ") diff --git a/internal/info/info_test.go b/internal/info/info_test.go index f8859ac1592..760284393fd 100644 --- a/internal/info/info_test.go +++ b/internal/info/info_test.go @@ -96,7 +96,7 @@ func TestString(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: "", + AlgorithmInfo: "", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -128,7 +128,7 @@ func TestString(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: "", + AlgorithmInfo: "", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -200,7 +200,7 @@ func TestGet(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: "", + AlgorithmInfo: "", BuildCPUInfoFlags: []string{""}, StackTrace: make([]StackTrace, 0, 10), }, @@ -270,7 +270,7 @@ func TestInit(t *testing.T) { version := Version buildTime := BuildTime cgoEnabled := CGOEnabled - ngtVersion := NGTVersion + ngtVersion := AlgorithmInfo buildCPUInfoFlags := BuildCPUInfoFlags tests := []test{ { @@ -282,16 +282,16 @@ func TestInit(t *testing.T) { want: &info{ baseURL: "https://github.com/vdaas/vald/tree/gitcommit", detail: Detail{ - GitCommit: "gitcommit", - ServerName: "gateway", - Version: "gitcommit", - BuildTime: "1s", - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoTrue, - NGTVersion: "v1.11.6", + GitCommit: "gitcommit", + ServerName: "gateway", + Version: "gitcommit", + BuildTime: "1s", + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoTrue, + AlgorithmInfo: "v1.11.6", BuildCPUInfoFlags: []string{ "avx512f", "avx512dq", }, @@ -311,7 +311,7 @@ func TestInit(t *testing.T) { Version = "" BuildTime = "1s" CGOEnabled = cgoTrue - NGTVersion = "v1.11.6" + AlgorithmInfo = "v1.11.6" BuildCPUInfoFlags = "\t\tavx512f avx512dq\t" }, afterFunc: func(t *testing.T, _ args) { @@ -323,7 +323,7 @@ func TestInit(t *testing.T) { Version = version BuildTime = buildTime CGOEnabled = cgoEnabled - NGTVersion = ngtVersion + AlgorithmInfo = ngtVersion BuildCPUInfoFlags = buildCPUInfoFlags }, }, @@ -336,16 +336,16 @@ func TestInit(t *testing.T) { want: &info{ baseURL: "https://github.com/vdaas/vald/tree/gitcommit", detail: Detail{ - GitCommit: "gitcommit", - ServerName: "", - Version: "gitcommit", - BuildTime: "1s", - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoTrue, - NGTVersion: "v1.11.6", + GitCommit: "gitcommit", + ServerName: "", + Version: "gitcommit", + BuildTime: "1s", + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoTrue, + AlgorithmInfo: "v1.11.6", BuildCPUInfoFlags: []string{ "avx512f", "avx512dq", }, @@ -365,7 +365,7 @@ func TestInit(t *testing.T) { Version = "" BuildTime = "1s" CGOEnabled = cgoTrue - NGTVersion = "v1.11.6" + AlgorithmInfo = "v1.11.6" BuildCPUInfoFlags = "\t\tavx512f avx512dq\t" }, afterFunc: func(t *testing.T, _ args) { @@ -377,7 +377,7 @@ func TestInit(t *testing.T) { Version = version BuildTime = buildTime CGOEnabled = cgoEnabled - NGTVersion = ngtVersion + AlgorithmInfo = ngtVersion BuildCPUInfoFlags = buildCPUInfoFlags }, }, @@ -463,7 +463,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -496,7 +496,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), }, prepOnce: func() (o sync.Once) { @@ -532,7 +532,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -567,7 +567,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -666,7 +666,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -689,7 +689,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -715,7 +715,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{}, }, @@ -734,7 +734,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -780,7 +780,7 @@ func TestDetail_String(t *testing.T) { GoOS string GoArch string CGOEnabled string - NGTVersion string + AlgorithmInfo string BuildCPUInfoFlags []string StackTrace []StackTrace } @@ -824,7 +824,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -845,7 +845,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -869,7 +869,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{}, }, @@ -883,7 +883,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -914,7 +914,7 @@ func TestDetail_String(t *testing.T) { GoOS: test.fields.GoOS, GoArch: test.fields.GoArch, CGOEnabled: test.fields.CGOEnabled, - NGTVersion: test.fields.NGTVersion, + AlgorithmInfo: test.fields.AlgorithmInfo, BuildCPUInfoFlags: test.fields.BuildCPUInfoFlags, StackTrace: test.fields.StackTrace, } @@ -960,17 +960,17 @@ func Test_info_Get(t *testing.T) { }, want: want{ want: Detail{ - ServerName: "", - Version: "", - GitCommit: GitCommit, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - StackTrace: []StackTrace{}, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + ServerName: "", + Version: "", + GitCommit: GitCommit, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + StackTrace: []StackTrace{}, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1015,8 +1015,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1061,8 +1061,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1106,8 +1106,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1151,8 +1151,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1196,8 +1196,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1241,8 +1241,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1318,15 +1318,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1348,15 +1348,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/internal", detail: Detail{ - GitCommit: "internal", - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: "internal", + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1378,15 +1378,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "v1.0.0", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "v1.0.0", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1408,15 +1408,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: "10s", - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: "10s", + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1438,15 +1438,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: "1.14", - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: "1.14", + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1468,15 +1468,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: "linux", - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: "linux", + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1498,15 +1498,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: "amd", - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: "amd", + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1528,15 +1528,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoTrue, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoTrue, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1558,15 +1558,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoFalse, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoFalse, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1578,25 +1578,25 @@ func Test_info_prepare(t *testing.T) { }, }, { - name: "set success with NGTVersion set", + name: "set success with AlgorithmInfo set", fields: fields{ detail: Detail{ - NGTVersion: "v1.11.5", + AlgorithmInfo: "v1.11.5", }, }, want: want{ want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: "v1.11.5", + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: "v1.11.5", BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1626,7 +1626,7 @@ func Test_info_prepare(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: []string{"avx512f"}, }, }, diff --git a/internal/net/dialer.go b/internal/net/dialer.go index 689d90f1167..8df5195e6f4 100644 --- a/internal/net/dialer.go +++ b/internal/net/dialer.go @@ -170,7 +170,7 @@ func (d *dialer) lookup(ctx context.Context, host string) (dc *dialerCache, err dc = &dialerCache{ ips: ips, } - log.Infof("lookup succeed %v", dc.ips) + log.Debugf("lookup succeed for %s, ips: %v", host, dc.ips) if d.enableDNSCache { d.dnsCache.Set(host, dc) } diff --git a/internal/net/grpc/admin/admin.go b/internal/net/grpc/admin/admin.go new file mode 100644 index 00000000000..bdf3e99719f --- /dev/null +++ b/internal/net/grpc/admin/admin.go @@ -0,0 +1,24 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package admin provides grpc admin metrics registration API for providing grpc metrics endpoints +package admin + +import ( + "google.golang.org/grpc/admin" +) + +var Register = admin.Register diff --git a/internal/net/grpc/errdetails/errdetails.go b/internal/net/grpc/errdetails/errdetails.go index 36503e69862..0b4bc4e4615 100644 --- a/internal/net/grpc/errdetails/errdetails.go +++ b/internal/net/grpc/errdetails/errdetails.go @@ -333,7 +333,7 @@ func DebugInfoFromInfoDetail(v *info.Detail) *DebugInfo { "Name:", v.ServerName, ",", "GitCommit:", v.GitCommit, ",", "BuildTime:", v.BuildTime, ",", - "NGT_Version:", v.NGTVersion, ",", + "Algorithm_Info:", v.AlgorithmInfo, ",", "Go_Version:", v.GoVersion, ",", "GOARCH:", v.GoArch, ",", "GOOS:", v.GoOS, ",", diff --git a/internal/net/grpc/interceptor/server/logging/accesslog.go b/internal/net/grpc/interceptor/server/logging/accesslog.go index e192749d82c..fdcc133d6dc 100644 --- a/internal/net/grpc/interceptor/server/logging/accesslog.go +++ b/internal/net/grpc/interceptor/server/logging/accesslog.go @@ -88,9 +88,9 @@ func AccessLogInterceptor() grpc.UnaryServerInterceptor { if err != nil { entity.Error = err - log.Infod(rpcCompletedMessage, entity) + log.Warn(rpcCompletedMessage, entity) } else { - log.Infod(rpcCompletedMessage, entity) + log.Debug(rpcCompletedMessage, entity) } return resp, err @@ -136,9 +136,9 @@ func AccessLogStreamInterceptor() grpc.StreamServerInterceptor { if err != nil { entity.Error = err - log.Infod(rpcCompletedMessage, entity) + log.Warn(rpcCompletedMessage, entity) } else { - log.Infod(rpcCompletedMessage, entity) + log.Debug(rpcCompletedMessage, entity) } return err diff --git a/internal/net/grpc/interceptor/server/metric/metric.go b/internal/net/grpc/interceptor/server/metric/metric.go index a2db800c3b8..3395ac7f749 100644 --- a/internal/net/grpc/interceptor/server/metric/metric.go +++ b/internal/net/grpc/interceptor/server/metric/metric.go @@ -36,7 +36,7 @@ const ( func MetricInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor, error) { meter := metrics.GetMeter() - latencyHistgram, err := meter.SyncFloat64().Histogram( + latencyHistgram, err := meter.Float64Histogram( latencyMetricsName, metrics.WithDescription("Server latency in milliseconds, by method"), metrics.WithUnit(metrics.Milliseconds), @@ -45,7 +45,7 @@ func MetricInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterce return nil, nil, errors.Wrap(err, "failed to create latency metric") } - completedRPCCnt, err := meter.SyncInt64().Counter( + completedRPCCnt, err := meter.Int64Counter( completedRPCsMetricsName, metrics.WithDescription("Count of RPCs by method and status"), metrics.WithUnit(metrics.Milliseconds), @@ -55,9 +55,9 @@ func MetricInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterce } record := func(ctx context.Context, method string, err error, latency float64) { - attrs := attributesFromError(method, err) - latencyHistgram.Record(ctx, latency, attrs...) - completedRPCCnt.Add(ctx, 1, attrs...) + opts := attribute.ToMeasurementOption(attributesFromError(method, err)...) + latencyHistgram.Record(ctx, latency, opts) + completedRPCCnt.Add(ctx, 1, opts) } return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { now := time.Now() diff --git a/internal/observability/attribute/attribute.go b/internal/observability/attribute/attribute.go index 24df7231e22..4f02d8a4106 100644 --- a/internal/observability/attribute/attribute.go +++ b/internal/observability/attribute/attribute.go @@ -13,13 +13,18 @@ // limitations under the License. package attribute -import "go.opentelemetry.io/otel/attribute" +import ( + "go.opentelemetry.io/otel/attribute" + api "go.opentelemetry.io/otel/metric" +) type ( KeyValue = attribute.KeyValue Key = attribute.Key ) +var ToMeasurementOption = api.WithAttributes + func Bool(k string, v bool) KeyValue { return attribute.Bool(k, v) } diff --git a/internal/observability/exporter/otlp/otlp.go b/internal/observability/exporter/otlp/otlp.go index 652d5d964fc..c09fa8f837f 100644 --- a/internal/observability/exporter/otlp/otlp.go +++ b/internal/observability/exporter/otlp/otlp.go @@ -27,12 +27,11 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/metric/global" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.12.0" + semconv "go.opentelemetry.io/otel/semconv/v1.20.0" ) // Metrics and Trace attribute keys. @@ -123,7 +122,8 @@ func (e *exp) initMeter(ctx context.Context) (err error) { e.metricsExporter, metric.WithInterval(e.mExportInterval), metric.WithTimeout(e.mExportTimeout), - ), e.metricsViews...), + )), + metric.WithView(e.metricsViews...), metric.WithResource(resource.NewWithAttributes( semconv.SchemaURL, e.attributes..., @@ -144,7 +144,7 @@ func (e *exp) Start(ctx context.Context) error { otel.SetTextMapPropagator( propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}), ) - global.SetMeterProvider(e.meterProvider) + otel.SetMeterProvider(e.meterProvider) return nil } diff --git a/internal/observability/metrics/agent/core/faiss/faiss.go b/internal/observability/metrics/agent/core/faiss/faiss.go new file mode 100644 index 00000000000..fb0a68c34c2 --- /dev/null +++ b/internal/observability/metrics/agent/core/faiss/faiss.go @@ -0,0 +1,267 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package faiss + +import ( + "context" + + "github.com/vdaas/vald/internal/observability/metrics" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/aggregation" +) + +const ( + indexCountMetricsName = "agent_core_faiss_index_count" + indexCountMetricsDescription = "Agent Faiss index count" + + uncommittedIndexCountMetricsName = "agent_core_faiss_uncommitted_index_count" + uncommittedIndexCountMetricsDescription = "Agent Faiss index count" + + insertVQueueCountMetricsName = "agent_core_faiss_insert_vqueue_count" + insertVQueueCountMetricsDescription = "Agent Faiss insert vqueue count" + + deleteVQueueCountMetricsName = "agent_core_faiss_delete_vqueue_count" + deleteVQueueCountMetricsDescription = "Agent Faiss delete vqueue count" + + completedCreateIndexTotalMetricsName = "agent_core_faiss_completed_create_index_total" + completedCreateIndexTotalMetricsDescription = "The cumulative count of completed create index execution" + + executedProactiveGCTotalMetricsName = "agent_core_faiss_executed_proactive_gc_total" + executedProactiveGCTotalMetricsDescription = "The cumulative count of proactive GC execution" + + isIndexingMetricsName = "agent_core_faiss_is_indexing" + isIndexingMetricsDescription = "Currently indexing or no" + + isSavingMetricsName = "agent_core_faiss_is_saving" + isSavingMetricsDescription = "Currently saving or not" + + trainCountMetricsName = "agent_core_faiss_train_count" + trainCountMetricsDescription = "Agent Faiss train count" +) + +type faissMetrics struct { + faiss service.Faiss +} + +func New(f service.Faiss) metrics.Metric { + return &faissMetrics{ + faiss: f, + } +} + +func (f *faissMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: indexCountMetricsName, + Description: indexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uncommittedIndexCountMetricsName, + Description: uncommittedIndexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: insertVQueueCountMetricsName, + Description: insertVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: deleteVQueueCountMetricsName, + Description: deleteVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: completedCreateIndexTotalMetricsName, + Description: completedCreateIndexTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: executedProactiveGCTotalMetricsName, + Description: executedProactiveGCTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isIndexingMetricsName, + Description: isIndexingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isSavingMetricsName, + Description: isSavingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: trainCountMetricsName, + Description: trainCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + }, nil +} + +func (f *faissMetrics) Register(m metrics.Meter) error { + indexCount, err := m.Int64ObservableGauge( + indexCountMetricsName, + metrics.WithDescription(indexCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + uncommittedIndexCount, err := m.Int64ObservableGauge( + uncommittedIndexCountMetricsName, + metrics.WithDescription(uncommittedIndexCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + insertVQueueCount, err := m.Int64ObservableGauge( + insertVQueueCountMetricsName, + metrics.WithDescription(insertVQueueCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + deleteVQueueCount, err := m.Int64ObservableGauge( + deleteVQueueCountMetricsName, + metrics.WithDescription(deleteVQueueCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + completedCreateIndexTotal, err := m.Int64ObservableGauge( + completedCreateIndexTotalMetricsName, + metrics.WithDescription(completedCreateIndexTotalMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + executedProactiveGCTotal, err := m.Int64ObservableGauge( + executedProactiveGCTotalMetricsName, + metrics.WithDescription(executedProactiveGCTotalMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + isIndexing, err := m.Int64ObservableGauge( + isIndexingMetricsName, + metrics.WithDescription(isIndexingMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + isSaving, err := m.Int64ObservableGauge( + isSavingMetricsName, + metrics.WithDescription(isSavingMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + trainCount, err := m.Int64ObservableGauge( + trainCountMetricsName, + metrics.WithDescription(trainCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { + var indexing int64 + if f.faiss.IsIndexing() { + indexing = 1 + } + var saving int64 + if f.faiss.IsSaving() { + saving = 1 + } + + o.ObserveInt64(indexCount, int64(f.faiss.Len())) + o.ObserveInt64(uncommittedIndexCount, int64(f.faiss.InsertVQueueBufferLen()+f.faiss.DeleteVQueueBufferLen())) + o.ObserveInt64(insertVQueueCount, int64(f.faiss.InsertVQueueBufferLen())) + o.ObserveInt64(deleteVQueueCount, int64(int64(f.faiss.DeleteVQueueBufferLen()))) + o.ObserveInt64(completedCreateIndexTotal, int64(f.faiss.NumberOfCreateIndexExecution())) + o.ObserveInt64(executedProactiveGCTotal, int64(f.faiss.NumberOfProactiveGCExecution())) + o.ObserveInt64(isIndexing, int64(indexing)) + o.ObserveInt64(isSaving, int64(saving)) + o.ObserveInt64(trainCount, int64(f.faiss.GetTrainSize())) + + return nil + }, + indexCount, + uncommittedIndexCount, + insertVQueueCount, + deleteVQueueCount, + completedCreateIndexTotal, + executedProactiveGCTotal, + isIndexing, + isSaving, + trainCount, + ) + return err +} diff --git a/internal/observability/metrics/agent/core/ngt/ngt.go b/internal/observability/metrics/agent/core/ngt/ngt.go index a7016d35121..b4be39fd881 100644 --- a/internal/observability/metrics/agent/core/ngt/ngt.go +++ b/internal/observability/metrics/agent/core/ngt/ngt.go @@ -18,8 +18,9 @@ import ( "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/pkg/agent/core/ngt/service" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -61,103 +62,85 @@ func New(n service.NGT) metrics.Metric { } } -func (n *ngtMetrics) View() ([]*metrics.View, error) { - indexCount, err := view.New( - view.MatchInstrumentName(indexCountMetricsName), - view.WithSetDescription(indexCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - uncommittedIndexCount, err := view.New( - view.MatchInstrumentName(uncommittedIndexCountMetricsName), - view.WithSetDescription(uncommittedIndexCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - insertVQueueCount, err := view.New( - view.MatchInstrumentName(insertVQueueCountMetricsName), - view.WithSetDescription(insertVQueueCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - deleteVQueueCount, err := view.New( - view.MatchInstrumentName(deleteVQueueCountMetricsName), - view.WithSetDescription(deleteVQueueCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - completedCreateIndexTotal, err := view.New( - view.MatchInstrumentName(completedCreateIndexTotalMetricsName), - view.WithSetDescription(completedCreateIndexTotalMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - executedProactiveGCTotal, err := view.New( - view.MatchInstrumentName(executedProactiveGCTotalMetricsName), - view.WithSetDescription(executedProactiveGCTotalMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - isIndexing, err := view.New( - view.MatchInstrumentName(isIndexingMetricsName), - view.WithSetDescription(isIndexingMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - isSaving, err := view.New( - view.MatchInstrumentName(isSavingMetricsName), - view.WithSetDescription(isSavingMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - brokenIndexCount, err := view.New( - view.MatchInstrumentName(brokenIndexStoreCountMetricsName), - view.WithSetDescription(brokenIndexStoreCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &indexCount, - &uncommittedIndexCount, - &insertVQueueCount, - &deleteVQueueCount, - &completedCreateIndexTotal, - &executedProactiveGCTotal, - &isIndexing, - &isSaving, - &brokenIndexCount, +func (n *ngtMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: indexCountMetricsName, + Description: indexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uncommittedIndexCountMetricsName, + Description: uncommittedIndexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: insertVQueueCountMetricsName, + Description: insertVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: deleteVQueueCountMetricsName, + Description: deleteVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: completedCreateIndexTotalMetricsName, + Description: completedCreateIndexTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: executedProactiveGCTotalMetricsName, + Description: executedProactiveGCTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isIndexingMetricsName, + Description: isIndexingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isSavingMetricsName, + Description: isSavingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (n *ngtMetrics) Register(m metrics.Meter) error { - indexCount, err := m.AsyncInt64().Gauge( + indexCount, err := m.Int64ObservableGauge( indexCountMetricsName, metrics.WithDescription(indexCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -166,7 +149,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - uncommittedIndexCount, err := m.AsyncInt64().Gauge( + uncommittedIndexCount, err := m.Int64ObservableGauge( uncommittedIndexCountMetricsName, metrics.WithDescription(uncommittedIndexCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -175,7 +158,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - insertVQueueCount, err := m.AsyncInt64().Gauge( + insertVQueueCount, err := m.Int64ObservableGauge( insertVQueueCountMetricsName, metrics.WithDescription(insertVQueueCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -184,7 +167,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - deleteVQueueCount, err := m.AsyncInt64().Gauge( + deleteVQueueCount, err := m.Int64ObservableGauge( deleteVQueueCountMetricsName, metrics.WithDescription(deleteVQueueCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -193,7 +176,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - completedCreateIndexTotal, err := m.AsyncInt64().Gauge( + completedCreateIndexTotal, err := m.Int64ObservableGauge( completedCreateIndexTotalMetricsName, metrics.WithDescription(completedCreateIndexTotalMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -202,7 +185,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - executedProactiveGCTotal, err := m.AsyncInt64().Gauge( + executedProactiveGCTotal, err := m.Int64ObservableGauge( executedProactiveGCTotalMetricsName, metrics.WithDescription(executedProactiveGCTotalMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -211,7 +194,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - isIndexing, err := m.AsyncInt64().Gauge( + isIndexing, err := m.Int64ObservableGauge( isIndexingMetricsName, metrics.WithDescription(isIndexingMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -220,7 +203,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - isSaving, err := m.AsyncInt64().Gauge( + isSaving, err := m.Int64ObservableGauge( isSavingMetricsName, metrics.WithDescription(isSavingMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -229,47 +212,34 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - brokenIndexCount, err := m.AsyncInt64().Gauge( - brokenIndexStoreCountMetricsName, - metrics.WithDescription(brokenIndexStoreCountMetricsDescription), - metrics.WithUnit(metrics.Dimensionless), - ) - if err != nil { - return err - } - - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - indexCount, - uncommittedIndexCount, - insertVQueueCount, - deleteVQueueCount, - completedCreateIndexTotal, - executedProactiveGCTotal, - isIndexing, - isSaving, - brokenIndexCount, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { var indexing int64 if n.ngt.IsIndexing() { indexing = 1 } - var saving int64 if n.ngt.IsSaving() { saving = 1 } - - indexCount.Observe(ctx, int64(n.ngt.Len())) - uncommittedIndexCount.Observe(ctx, int64(n.ngt.InsertVQueueBufferLen()+n.ngt.DeleteVQueueBufferLen())) - insertVQueueCount.Observe(ctx, int64(n.ngt.InsertVQueueBufferLen())) - deleteVQueueCount.Observe(ctx, int64(int64(n.ngt.DeleteVQueueBufferLen()))) - completedCreateIndexTotal.Observe(ctx, int64(n.ngt.NumberOfCreateIndexExecution())) - executedProactiveGCTotal.Observe(ctx, int64(n.ngt.NumberOfProactiveGCExecution())) - isIndexing.Observe(ctx, int64(indexing)) - isSaving.Observe(ctx, int64(saving)) - brokenIndexCount.Observe(ctx, int64(n.ngt.BrokenIndexCount())) + o.ObserveInt64(indexCount, int64(n.ngt.Len())) + o.ObserveInt64(uncommittedIndexCount, int64(n.ngt.InsertVQueueBufferLen()+n.ngt.DeleteVQueueBufferLen())) + o.ObserveInt64(insertVQueueCount, int64(n.ngt.InsertVQueueBufferLen())) + o.ObserveInt64(deleteVQueueCount, int64(int64(n.ngt.DeleteVQueueBufferLen()))) + o.ObserveInt64(completedCreateIndexTotal, int64(n.ngt.NumberOfCreateIndexExecution())) + o.ObserveInt64(executedProactiveGCTotal, int64(n.ngt.NumberOfProactiveGCExecution())) + o.ObserveInt64(isIndexing, int64(indexing)) + o.ObserveInt64(isSaving, int64(saving)) + return nil }, - ) + indexCount, + uncommittedIndexCount, + insertVQueueCount, + deleteVQueueCount, + completedCreateIndexTotal, + executedProactiveGCTotal, + isIndexing, + isSaving, + ) + return err } diff --git a/internal/observability/metrics/agent/sidecar/sidecar.go b/internal/observability/metrics/agent/sidecar/sidecar.go index 433e535a8e3..c013feec511 100644 --- a/internal/observability/metrics/agent/sidecar/sidecar.go +++ b/internal/observability/metrics/agent/sidecar/sidecar.go @@ -21,8 +21,9 @@ import ( "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/internal/sync" "github.com/vdaas/vald/pkg/agent/sidecar/service/observer" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -58,45 +59,42 @@ func New() MetricsHook { } } -func (*sidecarMetrics) View() ([]*metrics.View, error) { - uploadTotal, err := view.New( - view.MatchInstrumentName(uploadTotalMetricsName), - view.WithSetDescription(uploadTotalMetricsDescription), - view.WithSetAggregation(aggregation.Sum{}), - ) - if err != nil { - return nil, err - } - - uploadBytes, err := view.New( - view.MatchInstrumentName(uploadBytesMetricsName), - view.WithSetDescription(uploadBytesMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - uploadLatency, err := view.New( - view.MatchInstrumentName(uploadLatencyMetricsName), - view.WithSetDescription(uploadLatencyMetricsDescription), - view.WithSetAggregation(aggregation.ExplicitBucketHistogram{ - Boundaries: metrics.RoughMillisecondsDistribution, - }), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &uploadTotal, - &uploadBytes, - &uploadLatency, +func (*sidecarMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: uploadTotalMetricsName, + Description: uploadTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.Sum{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uploadBytesMetricsName, + Description: uploadBytesMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uploadLatencyMetricsName, + Description: uploadLatencyMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.ExplicitBucketHistogram{ + Boundaries: metrics.RoughMillisecondsDistribution, + }, + }, + ), }, nil } func (sm *sidecarMetrics) Register(m metrics.Meter) error { - uploadTotal, err := m.AsyncInt64().Counter( + uploadTotal, err := m.Int64ObservableCounter( uploadTotalMetricsName, metrics.WithDescription(uploadTotalMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -104,7 +102,7 @@ func (sm *sidecarMetrics) Register(m metrics.Meter) error { if err != nil { return err } - uploadBytes, err := m.AsyncInt64().Gauge( + uploadBytes, err := m.Int64ObservableGauge( uploadBytesMetricsName, metrics.WithDescription(uploadBytesMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -112,7 +110,7 @@ func (sm *sidecarMetrics) Register(m metrics.Meter) error { if err != nil { return err } - uploadLatency, err := m.AsyncFloat64().Gauge( + uploadLatency, err := m.Float64ObservableGauge( uploadLatencyMetricsName, metrics.WithDescription(uploadLatencyMetricsDescription), metrics.WithUnit(metrics.Milliseconds), @@ -121,35 +119,33 @@ func (sm *sidecarMetrics) Register(m metrics.Meter) error { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - uploadTotal, - uploadBytes, - uploadLatency, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { sm.mu.Lock() defer sm.mu.Unlock() if sm.info == nil { - return + return nil } - attrs := []attribute.KeyValue{ + opts := attribute.ToMeasurementOption( attribute.String(sm.storageTypeKey, sm.info.StorageInfo.Type), attribute.String(sm.bucketNameKey, sm.info.BucketName), - attribute.String(sm.filenameKey, sm.info.Filename), - } + attribute.String(sm.filenameKey, sm.info.Filename)) + o.ObserveInt64(uploadTotal, 1, opts) + o.ObserveInt64(uploadBytes, sm.info.Bytes, opts) latencyMillis := float64(sm.info.EndTime.Sub(sm.info.StartTime)) / float64(time.Millisecond) - - uploadTotal.Observe(ctx, 1, attrs...) - uploadBytes.Observe(ctx, sm.info.Bytes, attrs...) - uploadLatency.Observe(ctx, latencyMillis, attrs...) - + o.ObserveFloat64(uploadLatency, latencyMillis, opts) sm.info = nil + + return nil }, + uploadTotal, + uploadBytes, + uploadLatency, ) + return err } func (*sidecarMetrics) BeforeProcess(ctx context.Context, _ *observer.BackupInfo) (context.Context, error) { diff --git a/internal/observability/metrics/backoff/backoff.go b/internal/observability/metrics/backoff/backoff.go index c085b10ed3c..c24ff367525 100644 --- a/internal/observability/metrics/backoff/backoff.go +++ b/internal/observability/metrics/backoff/backoff.go @@ -19,8 +19,9 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -38,22 +39,22 @@ func New() metrics.Metric { } } -func (*backoffMetrics) View() ([]*metrics.View, error) { - retryCount, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &retryCount, +func (*backoffMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } -func (bm *backoffMetrics) Register(m metrics.Meter) error { - retryCount, err := m.AsyncInt64().Gauge( +func (bm *backoffMetrics) Register(m metrics.Meter) (err error) { + retryCount, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -61,18 +62,17 @@ func (bm *backoffMetrics) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - retryCount, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(ctx context.Context, o api.Observer) error { ms := backoff.Metrics(ctx) if len(ms) == 0 { - return + return nil } for name, cnt := range ms { - retryCount.Observe(ctx, cnt, attribute.String(bm.backoffNameKey, name)) + o.ObserveInt64(retryCount, cnt, attribute.ToMeasurementOption(attribute.String(bm.backoffNameKey, name))) } - }, + return nil + }, retryCount, ) + return err } diff --git a/internal/observability/metrics/circuitbreaker/circuitbreaker.go b/internal/observability/metrics/circuitbreaker/circuitbreaker.go index 64b5feb3c03..ff7ccc1a68f 100644 --- a/internal/observability/metrics/circuitbreaker/circuitbreaker.go +++ b/internal/observability/metrics/circuitbreaker/circuitbreaker.go @@ -19,8 +19,9 @@ import ( "github.com/vdaas/vald/internal/circuitbreaker" "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -40,23 +41,22 @@ func New() metrics.Metric { } } -func (*breakerMetrics) View() ([]*metrics.View, error) { - breakerState, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &breakerState, +func (*breakerMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (bm *breakerMetrics) Register(m metrics.Meter) error { - breakerState, err := m.AsyncInt64().Gauge( + breakerState, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -65,23 +65,22 @@ func (bm *breakerMetrics) Register(m metrics.Meter) error { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - breakerState, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(ctx context.Context, o api.Observer) error { ms := circuitbreaker.Metrics(ctx) if len(ms) != 0 { for name, sts := range ms { if len(sts) != 0 { for st, cnt := range sts { - breakerState.Observe(ctx, cnt, + o.ObserveInt64(breakerState, cnt, attribute.ToMeasurementOption( attribute.String(bm.breakerNameKey, name), - attribute.String(bm.stateKey, st.String())) + attribute.String(bm.stateKey, st.String()))) } } } } - }, + return nil + }, breakerState, ) + return err } diff --git a/internal/observability/metrics/grpc/grpc.go b/internal/observability/metrics/grpc/grpc.go index 3e8f8fa19f1..450e9bef2c5 100644 --- a/internal/observability/metrics/grpc/grpc.go +++ b/internal/observability/metrics/grpc/grpc.go @@ -15,8 +15,8 @@ package grpc import ( "github.com/vdaas/vald/internal/observability/metrics" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -33,29 +33,28 @@ func New() metrics.Metric { return &grpcServerMetrics{} } -func (*grpcServerMetrics) View() ([]*metrics.View, error) { - latencyHistgram, err := view.New( - view.MatchInstrumentName(latencyMetricsName), - view.WithSetDescription(latencyMetricsDesctiption), - view.WithSetAggregation(aggregation.ExplicitBucketHistogram{ - Boundaries: metrics.DefaultMillisecondsDistribution, - }), - ) - if err != nil { - return nil, err - } - - completedRPCCnt, err := view.New( - view.MatchInstrumentName(completedRPCsMetricsName), - view.WithSetDescription(completedRPCsMetricsDescription), - view.WithSetAggregation(aggregation.Sum{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &latencyHistgram, - &completedRPCCnt, +func (*grpcServerMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: latencyMetricsName, + Description: latencyMetricsDesctiption, + }, + view.Stream{ + Aggregation: aggregation.ExplicitBucketHistogram{ + Boundaries: metrics.DefaultMillisecondsDistribution, + }, + }, + ), + view.NewView( + view.Instrument{ + Name: completedRPCsMetricsName, + Description: completedRPCsMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.Sum{}, + }, + ), }, nil } diff --git a/internal/observability/metrics/info/info.go b/internal/observability/metrics/info/info.go index f6c8dbed02a..5c1cd230f5a 100644 --- a/internal/observability/metrics/info/info.go +++ b/internal/observability/metrics/info/info.go @@ -21,8 +21,9 @@ import ( "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) type info struct { @@ -83,23 +84,22 @@ func labelKVs(i interface{}) map[string]string { return kvs } -func (i *info) View() ([]*metrics.View, error) { - info, err := view.New( - view.MatchInstrumentName(i.name), - view.WithSetDescription(i.description), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &info, +func (i *info) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: i.name, + Description: i.description, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (i *info) Register(m metrics.Meter) error { - info, err := m.AsyncInt64().Gauge( + info, err := m.Int64ObservableGauge( i.name, metrics.WithDescription(i.description), metrics.WithUnit(metrics.Dimensionless), @@ -107,16 +107,15 @@ func (i *info) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - info, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { attrs := make([]attribute.KeyValue, 0, len(i.kvs)) for key, val := range i.kvs { attrs = append(attrs, attribute.String(key, val)) } - info.Observe(ctx, 1, attrs...) - }, + o.ObserveInt64(info, 1, attribute.ToMeasurementOption(attrs...)) + return nil + }, info, ) + return err } diff --git a/internal/observability/metrics/manager/index/index.go b/internal/observability/metrics/manager/index/index.go index 9976ab8095a..4e9dad756c9 100644 --- a/internal/observability/metrics/manager/index/index.go +++ b/internal/observability/metrics/manager/index/index.go @@ -18,8 +18,9 @@ import ( "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/pkg/manager/index/service" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -43,43 +44,40 @@ func New(i service.Indexer) metrics.Metric { } } -func (*indexerMetrics) View() ([]*metrics.View, error) { - uuidCount, err := view.New( - view.MatchInstrumentName(uuidCountMetricsName), - view.WithSetDescription(uuidCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - uncommittedUUIDCount, err := view.New( - view.MatchInstrumentName(uncommittedUUIDCountMetricsName), - view.WithSetDescription(uncommittedUUIDCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - isIndexing, err := view.New( - view.MatchInstrumentName(isIndexingMetricsName), - view.WithSetDescription(isIndexingMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &uuidCount, - &uncommittedUUIDCount, - &isIndexing, +func (*indexerMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: uuidCountMetricsName, + Description: uuidCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uncommittedUUIDCountMetricsName, + Description: uncommittedUUIDCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isIndexingMetricsName, + Description: isIndexingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (im *indexerMetrics) Register(m metrics.Meter) error { - uuidCount, err := m.AsyncInt64().Gauge( + uuidCount, err := m.Int64ObservableGauge( uuidCountMetricsName, metrics.WithDescription(uuidCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -88,7 +86,7 @@ func (im *indexerMetrics) Register(m metrics.Meter) error { return err } - uncommittedUUIDCount, err := m.AsyncInt64().Gauge( + uncommittedUUIDCount, err := m.Int64ObservableGauge( uncommittedUUIDCountMetricsName, metrics.WithDescription(uncommittedUUIDCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -97,7 +95,7 @@ func (im *indexerMetrics) Register(m metrics.Meter) error { return err } - isIndexing, err := m.AsyncInt64().Gauge( + isIndexing, err := m.Int64ObservableGauge( isIndexingMetricsName, metrics.WithDescription(isIndexingMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -106,20 +104,20 @@ func (im *indexerMetrics) Register(m metrics.Meter) error { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - uuidCount, - uncommittedUUIDCount, - isIndexing, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { var indexing int64 if im.indexer.IsIndexing() { indexing = 1 } - uuidCount.Observe(ctx, int64(im.indexer.NumberOfUUIDs())) - uncommittedUUIDCount.Observe(ctx, int64(im.indexer.NumberOfUncommittedUUIDs())) - isIndexing.Observe(ctx, int64(indexing)) + o.ObserveInt64(uuidCount, int64(im.indexer.NumberOfUUIDs())) + o.ObserveInt64(uncommittedUUIDCount, int64(im.indexer.NumberOfUncommittedUUIDs())) + o.ObserveInt64(isIndexing, int64(indexing)) + return nil }, + uuidCount, + uncommittedUUIDCount, + isIndexing, ) + return nil } diff --git a/internal/observability/metrics/mem/index/index.go b/internal/observability/metrics/mem/index/index.go new file mode 100644 index 00000000000..756f69e5925 --- /dev/null +++ b/internal/observability/metrics/mem/index/index.go @@ -0,0 +1,375 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package index + +import ( + "context" + "runtime" + "time" + + "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/aggregation" +) + +const ( + allocMetricsName = "alloc_bytes" + allocMetricsDescription = "Currently allocated number of bytes on the heap" + + totalAllocMetricsName = "alloc_bytes_total" + totalAllocMetricsDescription = "Cumulative bytes allocated for heap objects" + + sysMetricsName = "sys_bytes" + sysMetricsDescription = "Total bytes of memory obtained from the OS" + + mallocsMetricsName = "mallocs_total" + mallocsMetricsDescription = "The cumulative count of heap objects allocated" + + freesMetricsName = "frees_total" + freesMetricsDescription = "The cumulative count of heap objects freed" + + heapAllocMetricsName = "heap_alloc_bytes" + heapAllocMetricsDescription = "Bytes of allocated heap object" + + heapSysMetricsName = "heap_sys_bytes" + heapSysMetricsDescription = "Bytes of heap memory obtained from the OS" + + heapIdleMetricsName = "heap_idle_bytes" + heapIdleMetricsDescription = "Bytes in idle (unused) spans" + + heapInuseMetricsName = "heap_inuse_bytes" + heapInuseMetricsDescription = "Bytes in in-use spans" + + heapReleasedMetricsName = "heap_released_bytes" + heapReleasedMetricsDescription = "Bytes of physical memory returned to the OS" + + stackInuseMetricsName = "stack_inuse_bytes" + stackInuseMetricsDescription = "Bytes in stack spans" + + stackSysMetricsName = "stack_sys_bytes" + stackSysMetricsDescription = "Bytes of stack memory obtained from the OS" + + pauseTotalMsMetricsName = "pause_ms_total" + pauseTotalMsMetricsDescription = "The cumulative milliseconds in GC" + + numGCMetricsName = "gc_count" + numGCMetricsDescription = "The number of completed GC cycles" +) + +type memoryMetrics struct{} + +func New() metrics.Metric { + return &memoryMetrics{} +} + +func (*memoryMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: allocMetricsName, + Description: allocMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: totalAllocMetricsDescription, + Description: totalAllocMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: sysMetricsName, + Description: sysMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: mallocsMetricsName, + Description: mallocsMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: freesMetricsName, + Description: freesMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapAllocMetricsName, + Description: heapAllocMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapSysMetricsName, + Description: heapSysMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapIdleMetricsName, + Description: heapIdleMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapInuseMetricsName, + Description: heapInuseMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapReleasedMetricsName, + Description: heapReleasedMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: stackInuseMetricsName, + Description: stackInuseMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: stackSysMetricsName, + Description: stackSysMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: pauseTotalMsMetricsName, + Description: pauseTotalMsMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: numGCMetricsName, + Description: numGCMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + }, nil +} + +func (*memoryMetrics) Register(m metrics.Meter) error { + alloc, err := m.Int64ObservableGauge( + allocMetricsName, + metrics.WithDescription(allocMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + totalAlloc, err := m.Int64ObservableGauge( + totalAllocMetricsDescription, + metrics.WithDescription(totalAllocMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + sys, err := m.Int64ObservableGauge( + sysMetricsName, + metrics.WithDescription(sysMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + mallocs, err := m.Int64ObservableGauge( + mallocsMetricsName, + metrics.WithDescription(mallocsMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + frees, err := m.Int64ObservableGauge( + freesMetricsName, + metrics.WithDescription(freesMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + heapAlloc, err := m.Int64ObservableGauge( + heapAllocMetricsName, + metrics.WithDescription(heapAllocMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + heapSys, err := m.Int64ObservableGauge( + heapSysMetricsName, + metrics.WithDescription(heapSysMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + heapIdle, err := m.Int64ObservableGauge( + heapIdleMetricsName, + metrics.WithDescription(heapIdleMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + heapInuse, err := m.Int64ObservableGauge( + heapInuseMetricsName, + metrics.WithDescription(heapInuseMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + heapReleased, err := m.Int64ObservableGauge( + heapReleasedMetricsName, + metrics.WithDescription(heapReleasedMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + stackInuse, err := m.Int64ObservableGauge( + stackInuseMetricsName, + metrics.WithDescription(stackInuseMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + stackSys, err := m.Int64ObservableGauge( + stackSysMetricsName, + metrics.WithDescription(stackSysMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + pauseTotalMs, err := m.Int64ObservableGauge( + pauseTotalMsMetricsName, + metrics.WithDescription(pauseTotalMsMetricsDescription), + metrics.WithUnit(metrics.Milliseconds), + ) + if err != nil { + return err + } + + numGC, err := m.Int64ObservableGauge( + numGCMetricsName, + metrics.WithDescription(numGCMetricsDescription), + metrics.WithUnit(metrics.Bytes), + ) + if err != nil { + return err + } + + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { + var mstats runtime.MemStats + runtime.ReadMemStats(&mstats) + o.ObserveInt64(alloc, int64(mstats.Alloc)) + o.ObserveInt64(frees, int64(mstats.Frees)) + o.ObserveInt64(heapAlloc, int64(mstats.HeapAlloc)) + o.ObserveInt64(heapIdle, int64(mstats.HeapIdle)) + o.ObserveInt64(heapInuse, int64(mstats.HeapInuse)) + o.ObserveInt64(heapReleased, int64(mstats.HeapReleased)) + o.ObserveInt64(heapSys, int64(mstats.HeapSys)) + o.ObserveInt64(mallocs, int64(mstats.Mallocs)) + o.ObserveInt64(stackInuse, int64(mstats.StackInuse)) + o.ObserveInt64(stackSys, int64(mstats.StackSys)) + o.ObserveInt64(sys, int64(mstats.Sys)) + o.ObserveInt64(totalAlloc, int64(mstats.TotalAlloc)) + var ptMs int64 + if mstats.PauseTotalNs > 0 { + ptMs = int64(float64(mstats.PauseTotalNs) / float64(time.Millisecond)) + } + o.ObserveInt64(pauseTotalMs, ptMs) + o.ObserveInt64(numGC, int64(mstats.NextGC)) + return nil + }, + alloc, + frees, + heapAlloc, + heapIdle, + heapInuse, + heapReleased, + heapSys, + mallocs, + stackInuse, + stackSys, + sys, + totalAlloc, + pauseTotalMs, + numGC, + ) + return err +} diff --git a/internal/observability/metrics/metrics.go b/internal/observability/metrics/metrics.go index c6bf3e4cb03..7dec92e8aaa 100644 --- a/internal/observability/metrics/metrics.go +++ b/internal/observability/metrics/metrics.go @@ -14,16 +14,40 @@ package metrics import ( + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/unit" - "go.opentelemetry.io/otel/sdk/metric/view" + view "go.opentelemetry.io/otel/sdk/metric" ) -const ValdOrg = "vald.vdaas.org" +type ( + View = view.View + + // Meter is type alias of metrics.Meter. + Meter = metric.Meter + // Metric represents an interface for metric. + Metric interface { + View() ([]View, error) + Register(Meter) error + } +) + +const ( + ValdOrg = "vald.vdaas.org" + // Units defined by OpenTelemetry. + // Dimensionless is a type alias of unit.Dimensionless. + Dimensionless = "1" + // Bytes is a type alias of unit.Bytes. + Bytes = "By" + // Milliseconds is a type alias of unit.Milliseconds. + Milliseconds = "ms" +) var ( + // WithUnit returns an metric.WithUnit option. + WithUnit = metric.WithUnit + // WithDescription returns an metric.WithDescription option. + WithDescription = metric.WithDescription + RoughMillisecondsDistribution = []float64{ 1, 5, @@ -111,52 +135,7 @@ var ( } ) -// Meter is type alias of metrics.Meter. -type Meter = metric.Meter - // GetMeter returns the Meter object to record metrics. func GetMeter() Meter { - return global.MeterProvider().Meter(ValdOrg) -} - -// Unit is type alias of unit.Unit. -type Unit = unit.Unit - -// Units defined by OpenTelemetry. -const ( - // Dimensionless is a type alias of unit.Dimensionless. - Dimensionless = unit.Dimensionless - // Bytes is a type alias of unit.Bytes. - Bytes = unit.Bytes - // Milliseconds is a type alias of unit.Milliseconds. - Milliseconds = unit.Milliseconds -) - -type ( - // AsynchronousInstrument is type alias of instrument.Asynchronous. - AsynchronousInstrument = instrument.Asynchronous - // SynchronousInstrument is type alias of instrument.Synchronous. - SynchronousInstrument = instrument.Synchronous -) - -// WithUnit returns an instrument.WithUnit option. -func WithUnit(u Unit) instrument.Option { - return instrument.WithUnit(u) -} - -// WithDescription returns an instrument.WithDescription option. -func WithDescription(desc string) instrument.Option { - return instrument.WithDescription(desc) -} - -type View = view.View - -type Viewer interface { - View() ([]*View, error) -} - -// Metric represents an interface for metric. -type Metric interface { - Viewer - Register(Meter) error + return otel.GetMeterProvider().Meter(ValdOrg) } diff --git a/internal/observability/metrics/runtime/cgo/cgo.go b/internal/observability/metrics/runtime/cgo/cgo.go index 91550cbf26f..c3462dc1f7b 100644 --- a/internal/observability/metrics/runtime/cgo/cgo.go +++ b/internal/observability/metrics/runtime/cgo/cgo.go @@ -18,8 +18,9 @@ import ( "runtime" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -33,22 +34,22 @@ func New() metrics.Metric { return &cgo{} } -func (*cgo) View() ([]*metrics.View, error) { - count, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &count, +func (*cgo) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (*cgo) Register(m metrics.Meter) error { - count, err := m.AsyncInt64().Gauge( + count, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -56,12 +57,11 @@ func (*cgo) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - count, - }, - func(ctx context.Context) { - count.Observe(ctx, int64(runtime.NumGoroutine())) - }, + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { + o.ObserveInt64(count, runtime.NumCgoCall()) + return nil + }, count, ) + return err } diff --git a/internal/observability/metrics/runtime/goroutine/goroutine.go b/internal/observability/metrics/runtime/goroutine/goroutine.go index a20b11c9d4c..aee32fdf6aa 100644 --- a/internal/observability/metrics/runtime/goroutine/goroutine.go +++ b/internal/observability/metrics/runtime/goroutine/goroutine.go @@ -18,8 +18,9 @@ import ( "runtime" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -33,23 +34,22 @@ func New() metrics.Metric { return &goroutine{} } -func (*goroutine) View() ([]*metrics.View, error) { - count, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &count, +func (*goroutine) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } -func (*goroutine) Register(m metrics.Meter) error { - conter, err := m.AsyncInt64().Gauge( +func (*goroutine) Register(m metrics.Meter) (err error) { + counter, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -57,12 +57,11 @@ func (*goroutine) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - conter, - }, - func(ctx context.Context) { - conter.Observe(ctx, int64(runtime.NumGoroutine())) - }, + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { + o.ObserveInt64(counter, int64(runtime.NumGoroutine())) + return nil + }, counter, ) + return err } diff --git a/internal/observability/metrics/version/version.go b/internal/observability/metrics/version/version.go index afe6036f4ad..35ea0a7a1ab 100644 --- a/internal/observability/metrics/version/version.go +++ b/internal/observability/metrics/version/version.go @@ -22,8 +22,9 @@ import ( "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/internal/strings" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -76,22 +77,22 @@ func labelKVs(labels ...string) map[string]string { return info } -func (*version) View() ([]*metrics.View, error) { - otlv, err := view.New( - view.MatchInstrumentName(name), - view.WithSetDescription(description), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &otlv, +func (*version) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: name, + Description: description, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } -func (v *version) Register(m metrics.Meter) error { - info, err := m.AsyncInt64().Gauge( +func (v *version) Register(m metrics.Meter) (err error) { + info, err := m.Int64ObservableGauge( name, metrics.WithDescription(description), metrics.WithUnit(metrics.Dimensionless), @@ -99,16 +100,15 @@ func (v *version) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - info, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { attrs := make([]attribute.KeyValue, 0, len(v.kvs)) for key, val := range v.kvs { attrs = append(attrs, attribute.String(key, val)) } - info.Observe(ctx, 1, attrs...) - }, + o.ObserveInt64(info, 1, attribute.ToMeasurementOption(attrs...)) + return nil + }, info, ) + return err } diff --git a/internal/observability/trace/status.go b/internal/observability/trace/status.go index c1b08de2e43..7b324a49451 100644 --- a/internal/observability/trace/status.go +++ b/internal/observability/trace/status.go @@ -21,7 +21,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "go.opentelemetry.io/otel/attribute" ocodes "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.12.0" + semconv "go.opentelemetry.io/otel/semconv/v1.20.0" ) type Attributes = []attribute.KeyValue diff --git a/internal/sync/errgroup/group_test.go b/internal/sync/errgroup/group_test.go index 1e5161d0c3a..66eb33337a5 100644 --- a/internal/sync/errgroup/group_test.go +++ b/internal/sync/errgroup/group_test.go @@ -378,7 +378,14 @@ func Test_group_SetLimit(t *testing.T) { }, want: want{ want: &group{ +<<<<<<< HEAD:internal/sync/errgroup/group_test.go sem: semaphore.NewWeighted(0), +======= + enableLimitation: func() (el atomic.Bool) { + el.Store(false) + return el + }(), +>>>>>>> feature/agent/new-agents-helm-template-base:internal/errgroup/group_test.go }, }, }, @@ -391,7 +398,14 @@ func Test_group_SetLimit(t *testing.T) { fields: fields{}, want: want{ want: &group{ +<<<<<<< HEAD:internal/sync/errgroup/group_test.go sem: semaphore.NewWeighted(1), +======= + enableLimitation: func() (el atomic.Bool) { + el.Store(true) + return el + }(), +>>>>>>> feature/agent/new-agents-helm-template-base:internal/errgroup/group_test.go }, }, }, diff --git a/k8s/agent/daemonset.yaml b/k8s/agent/faiss/configmap.yaml similarity index 100% rename from k8s/agent/daemonset.yaml rename to k8s/agent/faiss/configmap.yaml diff --git a/k8s/agent/configmap.yaml b/k8s/agent/ngt/configmap.yaml similarity index 98% rename from k8s/agent/configmap.yaml rename to k8s/agent/ngt/configmap.yaml index e7345c38522..fb83571d166 100644 --- a/k8s/agent/configmap.yaml +++ b/k8s/agent/ngt/configmap.yaml @@ -17,7 +17,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: vald-agent-ngt-config + name: vald-agent-config labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.10 @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -181,7 +182,7 @@ data: namespace: "_MY_POD_NAMESPACE_" pod_name: "_MY_POD_NAME_" node_name: "_MY_NODE_NAME_" - service_name: "vald-agent-ngt" + service_name: "vald-agent" metrics: enable_cgo: true enable_goroutine: true @@ -195,7 +196,7 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false ngt: diff --git a/k8s/agent/pdb.yaml b/k8s/agent/pdb.yaml index 3dae150e8d5..7d314326b1d 100644 --- a/k8s/agent/pdb.yaml +++ b/k8s/agent/pdb.yaml @@ -17,7 +17,7 @@ apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: vald-agent-ngt + name: vald-agent labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.10 @@ -29,4 +29,4 @@ spec: maxUnavailable: 1 selector: matchLabels: - app: vald-agent-ngt + app: vald-agent diff --git a/k8s/agent/priorityclass.yaml b/k8s/agent/priorityclass.yaml index 2714ba13b98..e2679aa82c6 100644 --- a/k8s/agent/priorityclass.yaml +++ b/k8s/agent/priorityclass.yaml @@ -17,7 +17,7 @@ apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: - name: default-vald-agent-ngt-priority + name: default-vald-agent-priority labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.10 diff --git a/k8s/agent/deployment.yaml b/k8s/agent/qbg/configmap.yaml similarity index 100% rename from k8s/agent/deployment.yaml rename to k8s/agent/qbg/configmap.yaml diff --git a/k8s/agent/sidecar-configmap.yaml b/k8s/agent/sidecar/configmap.yaml similarity index 100% rename from k8s/agent/sidecar-configmap.yaml rename to k8s/agent/sidecar/configmap.yaml diff --git a/k8s/agent/sidecar-svc.yaml b/k8s/agent/sidecar/svc.yaml similarity index 100% rename from k8s/agent/sidecar-svc.yaml rename to k8s/agent/sidecar/svc.yaml diff --git a/k8s/agent/statefulset.yaml b/k8s/agent/statefulset.yaml index 0d0256c1691..b27bfd9d685 100644 --- a/k8s/agent/statefulset.yaml +++ b/k8s/agent/statefulset.yaml @@ -17,9 +17,9 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: vald-agent-ngt + name: vald-agent labels: - app: vald-agent-ngt + app: vald-agent app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.10 app.kubernetes.io/managed-by: Helm @@ -27,13 +27,13 @@ metadata: app.kubernetes.io/version: v1.7.10 app.kubernetes.io/component: agent spec: - serviceName: vald-agent-ngt + serviceName: vald-agent podManagementPolicy: Parallel replicas: 5 revisionHistoryLimit: 2 selector: matchLabels: - app: vald-agent-ngt + app: vald-agent updateStrategy: type: RollingUpdate rollingUpdate: @@ -42,7 +42,7 @@ spec: metadata: creationTimestamp: null labels: - app: vald-agent-ngt + app: vald-agent app.kubernetes.io/name: vald app.kubernetes.io/instance: release-name app.kubernetes.io/component: agent @@ -51,7 +51,7 @@ spec: profefe.com/port: "6060" profefe.com/service: vald-agent-ngt pyroscope.io/scrape: "true" - pyroscope.io/application-name: vald-agent-ngt + pyroscope.io/application-name: vald-agent pyroscope.io/profile-cpu-enabled: "true" pyroscope.io/profile-mem-enabled: "true" pyroscope.io/port: "6060" @@ -70,12 +70,12 @@ spec: - key: app operator: In values: - - vald-agent-ngt + - vald-agent topologyKey: kubernetes.io/hostname weight: 100 requiredDuringSchedulingIgnoredDuringExecution: [] containers: - - name: vald-agent-ngt + - name: vald-agent image: "vdaas/vald-agent-ngt:nightly" imagePullPolicy: Always livenessProbe: @@ -151,7 +151,7 @@ spec: fieldRef: fieldPath: metadata.namespace volumeMounts: - - name: vald-agent-ngt-config + - name: vald-agent-config mountPath: /etc/server/ dnsPolicy: ClusterFirst restartPolicy: Always @@ -164,9 +164,9 @@ spec: runAsUser: 65532 terminationGracePeriodSeconds: 120 volumes: - - name: vald-agent-ngt-config + - name: vald-agent-config configMap: defaultMode: 420 - name: vald-agent-ngt-config - priorityClassName: default-vald-agent-ngt-priority + name: vald-agent-config + priorityClassName: default-vald-agent-priority status: diff --git a/k8s/agent/svc.yaml b/k8s/agent/svc.yaml index dfd5c15eaef..0f1139ebe2c 100644 --- a/k8s/agent/svc.yaml +++ b/k8s/agent/svc.yaml @@ -17,7 +17,7 @@ apiVersion: v1 kind: Service metadata: - name: vald-agent-ngt + name: vald-agent labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.10 diff --git a/k8s/discoverer/configmap.yaml b/k8s/discoverer/configmap.yaml index b18fd3518a2..772ecb4b113 100644 --- a/k8s/discoverer/configmap.yaml +++ b/k8s/discoverer/configmap.yaml @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -195,7 +196,7 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false discoverer: @@ -215,7 +216,7 @@ data: pod_metrics: labels: {} fields: - containers.name: vald-agent-ngt + containers.name: vald-agent node_metrics: labels: {} fields: {} diff --git a/k8s/discoverer/deployment.yaml b/k8s/discoverer/deployment.yaml index 7b3ac4cd3b4..e8d14e721dc 100644 --- a/k8s/discoverer/deployment.yaml +++ b/k8s/discoverer/deployment.yaml @@ -47,7 +47,11 @@ spec: app.kubernetes.io/instance: release-name app.kubernetes.io/component: discoverer annotations: +<<<<<<< HEAD checksum/configmap: 6fd4f24bb970e1cbba59c4027add58cf9f9784f4f9465bbf040f6bd2f7149090 +======= + checksum/configmap: 6c2787f2ec13b72e346f247d33a9d4de8bb5aa8fe7c72e553cb63ef52c04287d +>>>>>>> feature/agent/new-agents-helm-template-base profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: vald-discoverer diff --git a/k8s/gateway/lb/configmap.yaml b/k8s/gateway/lb/configmap.yaml index d198de41f08..17a0ed74b8c 100644 --- a/k8s/gateway/lb/configmap.yaml +++ b/k8s/gateway/lb/configmap.yaml @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -195,13 +196,13 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false gateway: agent_port: 8081 - agent_name: "vald-agent-ngt" - agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_name: "vald-agent" + agent_dns: vald-agent.default.svc.cluster.local agent_namespace: "_MY_POD_NAMESPACE_" node_name: "" index_replica: 3 diff --git a/k8s/gateway/lb/deployment.yaml b/k8s/gateway/lb/deployment.yaml index 8f99c2961b3..94be779948c 100644 --- a/k8s/gateway/lb/deployment.yaml +++ b/k8s/gateway/lb/deployment.yaml @@ -46,7 +46,11 @@ spec: app.kubernetes.io/instance: release-name app.kubernetes.io/component: gateway-lb annotations: +<<<<<<< HEAD checksum/configmap: e097634475f923d5a9386f1ab3de8cedb8f3ba1f598228f3cec985df760d78c9 +======= + checksum/configmap: ddd9840cb18e6df32cbfbe8969eb3a0a25cfaf9906839d703be1fa6ad2c5d441 +>>>>>>> feature/agent/new-agents-helm-template-base profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: vald-lb-gateway @@ -75,7 +79,7 @@ spec: - -e - -c - | - until [ "$(wget --server-response --spider --quiet http://vald-agent-ngt.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + until [ "$(wget --server-response --spider --quiet http://vald-agent.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do echo "waiting for agent to be ready..." sleep 2; done diff --git a/k8s/gateway/lb/ing.yaml b/k8s/gateway/lb/ing.yaml index ed97d539c09..8e2bd32eca0 100644 --- a/k8s/gateway/lb/ing.yaml +++ b/k8s/gateway/lb/ing.yaml @@ -1 +1,47 @@ --- +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/grpc-backend: "true" + labels: + name: vald-lb-gateway-ingress + app: vald-lb-gateway-ingress + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.6 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.6 + app.kubernetes.io/component: gateway-lb + name: vald-lb-gateway-ingress +spec: + defaultBackend: + service: + name: vald-lb-gateway + port: + name: grpc + rules: + - host: lb.gateway.vald.vdaas.org + http: + paths: + - backend: + service: + name: vald-lb-gateway + port: + name: grpc + pathType: ImplementationSpecific diff --git a/k8s/manager/index/configmap.yaml b/k8s/manager/index/configmap.yaml index f2b4801d7c1..89213a925ec 100644 --- a/k8s/manager/index/configmap.yaml +++ b/k8s/manager/index/configmap.yaml @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -195,13 +196,13 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false indexer: agent_port: 8081 - agent_name: "vald-agent-ngt" - agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_name: "vald-agent" + agent_dns: vald-agent.default.svc.cluster.local agent_namespace: "_MY_POD_NAMESPACE_" node_name: "" discoverer: diff --git a/k8s/manager/index/deployment.yaml b/k8s/manager/index/deployment.yaml index 06d81b42849..9cf785c8ce6 100644 --- a/k8s/manager/index/deployment.yaml +++ b/k8s/manager/index/deployment.yaml @@ -47,7 +47,11 @@ spec: app.kubernetes.io/instance: release-name app.kubernetes.io/component: manager-index annotations: +<<<<<<< HEAD checksum/configmap: 6b725e1075416568b2458d912fea0c99f71f3fe3c06f6ce3f2cd62e4a6bb1127 +======= + checksum/configmap: b6519a6b4c7385fa9dc54a372afb7f8ce369624906e813b80d50656d6893999f +>>>>>>> feature/agent/new-agents-helm-template-base profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: vald-manager-index @@ -65,7 +69,7 @@ spec: - -e - -c - | - until [ "$(wget --server-response --spider --quiet http://vald-agent-ngt.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + until [ "$(wget --server-response --spider --quiet http://vald-agent.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do echo "waiting for agent to be ready..." sleep 2; done diff --git a/k8s/metrics/grafana/dashboards/01-vald-agent.yaml b/k8s/metrics/grafana/dashboards/01-vald-agent.yaml index 5f6e25b2f4b..691b1496a53 100644 --- a/k8s/metrics/grafana/dashboards/01-vald-agent.yaml +++ b/k8s/metrics/grafana/dashboards/01-vald-agent.yaml @@ -1066,7 +1066,7 @@ data: "calcs": [ "lastNotNull" ], - "fields": "/^ngt_version$/", + "fields": "/^algorithm_info$/", "values": false }, "text": {}, @@ -1080,7 +1080,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "label_replace(app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}, \"ngt_version\", \"v$1\", \"ngt_version\", \"([^v].*)\")", + "expr": "label_replace(app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}, \"algorithm_info\", \"v$1\", \"algorithm_info\", \"([^v].*)\")", "format": "table", "instant": true, "interval": "", diff --git a/k8s/operator/helm/crds/valdrelease.yaml b/k8s/operator/helm/crds/valdrelease.yaml index 08832a86b2f..3587c314a46 100644 --- a/k8s/operator/helm/crds/valdrelease.yaml +++ b/k8s/operator/helm/crds/valdrelease.yaml @@ -116,6 +116,12 @@ spec: items: type: object x-kubernetes-preserve-unknown-fields: true + algorithm: + type: string + enum: + - ngt + - qbg + - faiss annotations: type: object x-kubernetes-preserve-unknown-fields: true @@ -317,7 +323,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -758,6 +764,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -1248,7 +1256,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -1618,6 +1626,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2098,7 +2108,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -2459,6 +2469,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2918,7 +2930,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -3316,6 +3328,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -4279,7 +4293,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -4677,6 +4691,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -5438,7 +5454,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -5836,6 +5852,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -8518,7 +8536,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -10521,6 +10539,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: diff --git a/pkg/agent/core/faiss/config/config.go b/pkg/agent/core/faiss/config/config.go new file mode 100644 index 00000000000..8952abd7157 --- /dev/null +++ b/pkg/agent/core/faiss/config/config.go @@ -0,0 +1,77 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package setting stores all server application settings +package config + +import ( + "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" +) + +// GlobalConfig is type alias for config.GlobalConfig. +type GlobalConfig = config.GlobalConfig + +// Data represent a application setting data content (config.yaml). +// In K8s environment, this configuration is stored in K8s ConfigMap. +type Data struct { + GlobalConfig `json:",inline" yaml:",inline"` + + // Server represent all server configurations + Server *config.Servers `json:"server_config" yaml:"server_config"` + + // Observability represent observability configurations + Observability *config.Observability `json:"observability" yaml:"observability"` + + // Faiss represent faiss core configuration + Faiss *config.Faiss `json:"faiss" yaml:"faiss"` +} + +// NewConfig returns the Data struct or error from the given file path. +func NewConfig(path string) (cfg *Data, err error) { + cfg = new(Data) + + err = config.Read(path, &cfg) + if err != nil { + return nil, err + } + + if cfg != nil { + cfg.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Server != nil { + cfg.Server = cfg.Server.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Observability != nil { + cfg.Observability = cfg.Observability.Bind() + } else { + cfg.Observability = new(config.Observability).Bind() + } + + if cfg.Faiss != nil { + cfg.Faiss = cfg.Faiss.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + return cfg, nil +} diff --git a/pkg/agent/core/faiss/config/config_test.go b/pkg/agent/core/faiss/config/config_test.go new file mode 100644 index 00000000000..39331695b56 --- /dev/null +++ b/pkg/agent/core/faiss/config/config_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestNewConfig(t *testing.T) { +// type args struct { +// path string +// } +// type want struct { +// wantCfg *Data +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, *Data, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCfg *Data, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCfg, w.wantCfg) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCfg, w.wantCfg) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotCfg, err := NewConfig(test.args.path) +// if err := checkFunc(test.want, gotCfg, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/doc.go b/pkg/agent/core/faiss/handler/doc.go new file mode 100644 index 00000000000..825bcc7f61f --- /dev/null +++ b/pkg/agent/core/faiss/handler/doc.go @@ -0,0 +1,17 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package handler diff --git a/pkg/agent/core/faiss/handler/grpc/handler.go b/pkg/agent/core/faiss/handler/grpc/handler.go new file mode 100644 index 00000000000..89debe0b090 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/handler.go @@ -0,0 +1,95 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package grpc provides grpc server logic +package grpc + +import ( + "reflect" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" +) + +type Server interface { + agent.AgentServer + vald.Server +} + +type server struct { + name string + ip string + faiss service.Faiss + eg errgroup.Group + streamConcurrency int + agent.UnimplementedAgentServer + vald.UnimplementedValdServer +} + +const ( + apiName = "vald/agent/core/faiss" + faissResourceType = "vald/internal/core/algorithm" +) + +var errFaiss = new(errors.FaissError) + +func New(opts ...Option) (Server, error) { + s := new(server) + + for _, opt := range append(defaultOptions, opts...) { + if err := opt(s); err != nil { + werr := errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + + e := new(errors.ErrCriticalOption) + if errors.As(err, &e) { + log.Error(werr) + return nil, werr + } + log.Warn(werr) + } + } + return s, nil +} + +func (s *server) newLocations(uuids ...string) (locs *payload.Object_Locations) { + if len(uuids) == 0 { + return nil + } + locs = &payload.Object_Locations{ + Locations: make([]*payload.Object_Location, 0, len(uuids)), + } + for _, uuid := range uuids { + locs.Locations = append(locs.GetLocations(), &payload.Object_Location{ + Name: s.name, + Uuid: uuid, + Ips: []string{s.ip}, + }) + } + return locs +} + +func (s *server) newLocation(uuid string) *payload.Object_Location { + locs := s.newLocations(uuid) + if locs != nil && locs.GetLocations() != nil && len(locs.GetLocations()) > 0 { + return locs.Locations[0] + } + return nil +} diff --git a/pkg/agent/core/faiss/handler/grpc/handler_test.go b/pkg/agent/core/faiss/handler/grpc/handler_test.go new file mode 100644 index 00000000000..aa1865f9641 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/handler_test.go @@ -0,0 +1,352 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// opts []Option +// } +// type want struct { +// want Server +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Server, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Server, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, err := New(test.args.opts...) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_newLocations(t *testing.T) { +// type args struct { +// uuids []string +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantLocs *payload.Object_Locations +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotLocs *payload.Object_Locations) error { +// if !reflect.DeepEqual(gotLocs, w.wantLocs) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotLocs, w.wantLocs) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuids:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuids:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotLocs := s.newLocations(test.args.uuids...) +// if err := checkFunc(test.want, gotLocs); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_newLocation(t *testing.T) { +// type args struct { +// uuid string +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// want *payload.Object_Location +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Location) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got *payload.Object_Location) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// got := s.newLocation(test.args.uuid) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/index.go b/pkg/agent/core/faiss/handler/grpc/index.go new file mode 100644 index 00000000000..0ea6995e8ca --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/index.go @@ -0,0 +1,180 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" +) + +func (s *server) CreateIndex(ctx context.Context, c *payload.Control_CreateIndexRequest) (res *payload.Empty, err error) { + ctx, span := trace.StartSpan(ctx, apiName+".CreateIndex") + defer func() { + if span != nil { + span.End() + } + }() + res = new(payload.Empty) + err = s.faiss.CreateIndex(ctx) + if err != nil { + if errors.Is(err, errors.ErrUncommittedIndexNotFound) { + err = status.WrapWithFailedPrecondition(fmt.Sprintf("CreateIndex API failed"), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, + &errdetails.PreconditionFailure{ + Violations: []*errdetails.PreconditionFailureViolation{ + { + Type: "uncommitted index is empty", + Subject: "failed to CreateIndex operation caused by empty uncommitted indices", + }, + }, + }, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeFailedPrecondition(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + log.Error(err) + err = status.WrapWithInternal(fmt.Sprintf("CreateIndex API failed"), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInternal(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return res, nil +} + +func (s *server) SaveIndex(ctx context.Context, _ *payload.Empty) (res *payload.Empty, err error) { + ctx, span := trace.StartSpan(ctx, apiName+".SaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + res = new(payload.Empty) + err = s.faiss.SaveIndex(ctx) + if err != nil { + log.Error(err) + err = status.WrapWithInternal("SaveIndex API failed to save indices", err, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.SaveIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInternal(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return res, nil +} + +func (s *server) CreateAndSaveIndex(ctx context.Context, c *payload.Control_CreateIndexRequest) (res *payload.Empty, err error) { + ctx, span := trace.StartSpan(ctx, apiName+".CreateAndSaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + res = new(payload.Empty) + err = s.faiss.CreateAndSaveIndex(ctx) + if err != nil { + if errors.Is(err, errors.ErrUncommittedIndexNotFound) { + err = status.WrapWithFailedPrecondition(fmt.Sprintf("CreateAndSaveIndex API failed to create indexes pool_size = %d", c.GetPoolSize()), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateAndSaveIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, + &errdetails.PreconditionFailure{ + Violations: []*errdetails.PreconditionFailureViolation{ + { + Type: "uncommitted index is empty", + Subject: "failed to CreateAndSaveIndex operation caused by empty uncommitted indices", + }, + }, + }, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeFailedPrecondition(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + err = status.WrapWithInternal(fmt.Sprintf("CreateAndSaveIndex API failed to create indexes pool_size = %d", c.GetPoolSize()), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateAndSaveIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInternal(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return res, nil +} + +func (s *server) IndexInfo(ctx context.Context, c *payload.Empty) (res *payload.Info_Index_Count, err error) { + _, span := trace.StartSpan(ctx, apiName+".IndexInfo") + defer func() { + if span != nil { + span.End() + } + }() + + return &payload.Info_Index_Count{ + Stored: uint32(s.faiss.Len()), + Uncommitted: uint32(s.faiss.InsertVQueueBufferLen() + s.faiss.DeleteVQueueBufferLen()), + Indexing: s.faiss.IsIndexing(), + Saving: s.faiss.IsSaving(), + }, nil +} diff --git a/pkg/agent/core/faiss/handler/grpc/index_test.go b/pkg/agent/core/faiss/handler/grpc/index_test.go new file mode 100644 index 00000000000..adc819a9052 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/index_test.go @@ -0,0 +1,536 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_CreateIndex(t *testing.T) { +// type args struct { +// ctx context.Context +// c *payload.Control_CreateIndexRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Empty +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Empty, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Empty, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// c:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// c:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.CreateIndex(test.args.ctx, test.args.c) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_SaveIndex(t *testing.T) { +// type args struct { +// ctx context.Context +// in1 *payload.Empty +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Empty +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Empty, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Empty, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// in1:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// in1:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.SaveIndex(test.args.ctx, test.args.in1) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_CreateAndSaveIndex(t *testing.T) { +// type args struct { +// ctx context.Context +// c *payload.Control_CreateIndexRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Empty +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Empty, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Empty, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// c:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// c:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.CreateAndSaveIndex(test.args.ctx, test.args.c) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_IndexInfo(t *testing.T) { +// type args struct { +// ctx context.Context +// c *payload.Empty +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Info_Index_Count +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Info_Index_Count, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Info_Index_Count, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// c:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// c:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.IndexInfo(test.args.ctx, test.args.c) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/insert.go b/pkg/agent/core/faiss/handler/grpc/insert.go new file mode 100644 index 00000000000..44087a9deac --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/insert.go @@ -0,0 +1,141 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Insert(ctx context.Context, req *payload.Insert_Request) (res *payload.Object_Location, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.InsertRPCName) + defer func() { + if span != nil { + span.End() + } + }() + vec := req.GetVector() + if len(vec.GetVector()) != s.faiss.GetDimensionSize() { + err = errors.ErrIncompatibleDimensionSize(len(vec.GetVector()), int(s.faiss.GetDimensionSize())) + err = status.WrapWithInvalidArgument("Insert API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: vec.GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + err = s.faiss.InsertWithTime(vec.GetId(), vec.GetVector(), req.GetConfig().GetTimestamp()) + if err != nil { + var attrs []attribute.KeyValue + + if errors.Is(err, errors.ErrUUIDAlreadyExists(vec.GetId())) { + err = status.WrapWithAlreadyExists(fmt.Sprintf("Insert API uuid %s already exists", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeAlreadyExists(err.Error()) + } else if errors.Is(err, errors.ErrUUIDNotFound(0)) { + err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API empty uuid \"%s\" was given", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + } else { + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Internal, + "failed to parse Insert gRPC error response", + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + attrs = trace.FromGRPCStatus(st.Code(), msg) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return s.newLocation(vec.GetId()), nil +} + +func (s *server) StreamInsert(stream vald.Insert_StreamInsertServer) (err error) { + return s.UnimplementedValdServer.UnimplementedInsertServer.StreamInsert(stream) +} + +func (s *server) MultiInsert(ctx context.Context, reqs *payload.Insert_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedInsertServer.MultiInsert(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/insert_test.go b/pkg/agent/core/faiss/handler/grpc/insert_test.go new file mode 100644 index 00000000000..921b5842775 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/insert_test.go @@ -0,0 +1,399 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_Insert(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Insert_Request +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Location +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Location, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Location, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.Insert(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamInsert(t *testing.T) { +// type args struct { +// stream vald.Insert_StreamInsertServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamInsert(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiInsert(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Insert_MultiRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiInsert(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/linear_search.go b/pkg/agent/core/faiss/handler/grpc/linear_search.go new file mode 100644 index 00000000000..3fb17d836ec --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/linear_search.go @@ -0,0 +1,48 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" +) + +func (s *server) LinearSearch(ctx context.Context, req *payload.Search_Request) (res *payload.Search_Response, err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.LinearSearch(ctx, req) +} + +func (s *server) LinearSearchByID(ctx context.Context, req *payload.Search_IDRequest) (res *payload.Search_Response, err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.LinearSearchByID(ctx, req) +} + +func (s *server) StreamLinearSearch(stream vald.Search_StreamLinearSearchServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamLinearSearch(stream) +} + +func (s *server) StreamLinearSearchByID(stream vald.Search_StreamLinearSearchByIDServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamLinearSearchByID(stream) +} + +func (s *server) MultiLinearSearch(ctx context.Context, reqs *payload.Search_MultiRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiLinearSearch(ctx, reqs) +} + +func (s *server) MultiLinearSearchByID(ctx context.Context, reqs *payload.Search_MultiIDRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiLinearSearchByID(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/linear_search_test.go b/pkg/agent/core/faiss/handler/grpc/linear_search_test.go new file mode 100644 index 00000000000..9090d898ed9 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/linear_search_test.go @@ -0,0 +1,782 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_LinearSearch(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Search_Request +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Response +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Response, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.LinearSearch(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_LinearSearchByID(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Search_IDRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Response +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Response, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.LinearSearchByID(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamLinearSearch(t *testing.T) { +// type args struct { +// stream vald.Search_StreamLinearSearchServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamLinearSearch(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamLinearSearchByID(t *testing.T) { +// type args struct { +// stream vald.Search_StreamLinearSearchByIDServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamLinearSearchByID(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiLinearSearch(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Search_MultiRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Responses +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Responses, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Responses, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiLinearSearch(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiLinearSearchByID(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Search_MultiIDRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Responses +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Responses, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Responses, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiLinearSearchByID(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/object.go b/pkg/agent/core/faiss/handler/grpc/object.go new file mode 100644 index 00000000000..ce3e4bbdfaf --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/object.go @@ -0,0 +1,96 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" +) + +func (s *server) Exists(ctx context.Context, uid *payload.Object_ID) (res *payload.Object_ID, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.ExistsRPCName) + defer func() { + if span != nil { + span.End() + } + }() + uuid := uid.GetId() + if len(uuid) == 0 { + err = errors.ErrInvalidUUID(uuid) + err = status.WrapWithInvalidArgument(fmt.Sprintf("Exists API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(uid), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Exists", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + log.Warn(err) + return nil, err + } + if _, ok := s.faiss.Exists(uuid); !ok { + err = errors.ErrObjectIDNotFound(uid.GetId()) + err = status.WrapWithNotFound(fmt.Sprintf("Exists API meta %s's uuid not found", uid.GetId()), err, + &errdetails.RequestInfo{ + RequestId: uid.GetId(), + ServingData: errdetails.Serialize(uid), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Exists", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, + uid.GetId()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeNotFound(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return uid, nil +} + +func (s *server) GetObject(ctx context.Context, id *payload.Object_VectorRequest) (res *payload.Object_Vector, err error) { + return s.UnimplementedValdServer.UnimplementedObjectServer.GetObject(ctx, id) +} + +func (s *server) StreamGetObject(stream vald.Object_StreamGetObjectServer) (err error) { + return s.UnimplementedValdServer.UnimplementedObjectServer.StreamGetObject(stream) +} diff --git a/pkg/agent/core/faiss/handler/grpc/object_test.go b/pkg/agent/core/faiss/handler/grpc/object_test.go new file mode 100644 index 00000000000..4a6c82b00d9 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/object_test.go @@ -0,0 +1,399 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_Exists(t *testing.T) { +// type args struct { +// ctx context.Context +// uid *payload.Object_ID +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_ID +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_ID, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_ID, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// uid:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// uid:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.Exists(test.args.ctx, test.args.uid) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_GetObject(t *testing.T) { +// type args struct { +// ctx context.Context +// id *payload.Object_VectorRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Vector +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Vector, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Vector, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// id:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// id:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.GetObject(test.args.ctx, test.args.id) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamGetObject(t *testing.T) { +// type args struct { +// stream vald.Object_StreamGetObjectServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamGetObject(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/option.go b/pkg/agent/core/faiss/handler/grpc/option.go new file mode 100644 index 00000000000..2b443b1cfe4 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/option.go @@ -0,0 +1,100 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package grpc provides grpc server logic +package grpc + +import ( + "os" + "runtime" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" +) + +// Option represents the functional option for server. +type Option func(*server) error + +var defaultOptions = []Option{ + WithName(func() string { + name, err := os.Hostname() + if err != nil { + log.Warn(err) + } + return name + }()), + WithIP(net.LoadLocalIP()), + WithStreamConcurrency(runtime.GOMAXPROCS(-1) * 10), + WithErrGroup(errgroup.Get()), +} + +// WithIP returns the option to set the IP for server. +func WithIP(ip string) Option { + return func(s *server) error { + if len(ip) == 0 { + return errors.NewErrInvalidOption("ip", ip) + } + s.ip = ip + return nil + } +} + +// WithName returns the option to set the name for server. +func WithName(name string) Option { + return func(s *server) error { + if len(name) == 0 { + return errors.NewErrInvalidOption("name", name) + } + s.name = name + return nil + } +} + +// WithFaiss returns the option to set the Faiss service for server. +func WithFaiss(f service.Faiss) Option { + return func(s *server) error { + if f == nil { + return errors.NewErrInvalidOption("faiss", f) + } + s.faiss = f + return nil + } +} + +// WithStreamConcurrency returns the option to set the stream concurrency for server. +func WithStreamConcurrency(c int) Option { + return func(s *server) error { + if c <= 0 { + return errors.NewErrInvalidOption("streamConcurrency", c) + } + s.streamConcurrency = c + return nil + } +} + +// WithErrGroup returns the option to set the error group for server. +func WithErrGroup(eg errgroup.Group) Option { + return func(s *server) error { + if eg == nil { + return errors.NewErrInvalidOption("errGroup", eg) + } + s.eg = eg + return nil + } +} diff --git a/pkg/agent/core/faiss/handler/grpc/option_test.go b/pkg/agent/core/faiss/handler/grpc/option_test.go new file mode 100644 index 00000000000..7bc7ce76f65 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/option_test.go @@ -0,0 +1,446 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func TestWithIP(t *testing.T) { +// type args struct { +// ip string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ip:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ip:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithIP(test.args.ip) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithName(t *testing.T) { +// type args struct { +// name string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// name:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// name:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithName(test.args.name) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithFaiss(t *testing.T) { +// type args struct { +// f service.Faiss +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// f:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// f:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithFaiss(test.args.f) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithStreamConcurrency(t *testing.T) { +// type args struct { +// c int +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// c:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// c:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithStreamConcurrency(test.args.c) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithErrGroup(t *testing.T) { +// type args struct { +// eg errgroup.Group +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// eg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// eg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithErrGroup(test.args.eg) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/remove.go b/pkg/agent/core/faiss/handler/grpc/remove.go new file mode 100644 index 00000000000..96b858de914 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/remove.go @@ -0,0 +1,134 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Remove(ctx context.Context, req *payload.Remove_Request) (res *payload.Object_Location, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.RemoveRPCName) + defer func() { + if span != nil { + span.End() + } + }() + id := req.GetId() + uuid := id.GetId() + if len(uuid) == 0 { + err = errors.ErrInvalidUUID(uuid) + err = status.WrapWithInvalidArgument(fmt.Sprintf("Remove API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + log.Warn(err) + return nil, err + } + err = s.faiss.DeleteWithTime(uuid, req.GetConfig().GetTimestamp()) + if err != nil { + var attrs []attribute.KeyValue + if errors.Is(err, errors.ErrObjectIDNotFound(uuid)) { + err = status.WrapWithNotFound(fmt.Sprintf("Remove API uuid %s not found", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeNotFound(err.Error()) + } else if errors.Is(err, errors.ErrUUIDNotFound(0)) { + err = status.WrapWithInvalidArgument(fmt.Sprintf("Remove API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + } else { + err = status.WrapWithInternal("Remove API failed", err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return s.newLocation(uuid), nil +} + +func (s *server) StreamRemove(stream vald.Remove_StreamRemoveServer) (err error) { + return s.UnimplementedValdServer.UnimplementedRemoveServer.StreamRemove(stream) +} + +func (s *server) MultiRemove(ctx context.Context, reqs *payload.Remove_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedRemoveServer.MultiRemove(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/remove_test.go b/pkg/agent/core/faiss/handler/grpc/remove_test.go new file mode 100644 index 00000000000..62f3f64081a --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/remove_test.go @@ -0,0 +1,399 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_Remove(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Remove_Request +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Location +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Location, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Location, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.Remove(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamRemove(t *testing.T) { +// type args struct { +// stream vald.Remove_StreamRemoveServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamRemove(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiRemove(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Remove_MultiRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiRemove(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/search.go b/pkg/agent/core/faiss/handler/grpc/search.go new file mode 100644 index 00000000000..8aecfb6e7a8 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/search.go @@ -0,0 +1,196 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/pkg/agent/core/faiss/model" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Search(ctx context.Context, req *payload.Search_Request) (res *payload.Search_Response, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.SearchRPCName) + defer func() { + if span != nil { + span.End() + } + }() + if len(req.GetVector()) != s.faiss.GetDimensionSize() { + err = errors.ErrIncompatibleDimensionSize(len(req.GetVector()), int(s.faiss.GetDimensionSize())) + err = status.WrapWithInvalidArgument("Search API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + }) + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + res, err = toSearchResponse( + s.faiss.Search( + req.GetConfig().GetNum(), + 1, + req.GetVector())) + if err != nil || res == nil { + var attrs []attribute.KeyValue + switch { + case errors.Is(err, errors.ErrCreateIndexingIsInProgress): + err = status.WrapWithAborted("Search API aborted to process search request due to createing indices is in progress", err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Debug(err) + attrs = trace.StatusCodeAborted(err.Error()) + case errors.Is(err, errors.ErrEmptySearchResult), + err == nil && res == nil, + 0 < req.GetConfig().GetMinNum() && len(res.GetResults()) < int(req.GetConfig().GetMinNum()): + err = status.WrapWithNotFound(fmt.Sprintf("Search API requestID %s's search result not found", req.GetConfig().GetRequestId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Debug(err) + attrs = trace.StatusCodeNotFound(err.Error()) + case errors.As(err, &errFaiss): + log.Errorf("faiss core process returned error: %v", err) + err = status.WrapWithInternal("Search API failed to process search request due to faiss core process returned error", err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search/core.faiss", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + case errors.Is(err, errors.ErrIncompatibleDimensionSize(len(req.GetVector()), int(s.faiss.GetDimensionSize()))): + err = status.WrapWithInvalidArgument("Search API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + default: + err = status.WrapWithInternal("Search API failed to process search request", err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + res.RequestId = req.GetConfig().GetRequestId() + return res, nil +} + +func (s *server) SearchByID(ctx context.Context, req *payload.Search_IDRequest) (res *payload.Search_Response, err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.SearchByID(ctx, req) +} + +func (s *server) StreamSearch(stream vald.Search_StreamSearchServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamSearch(stream) +} + +func (s *server) StreamSearchByID(stream vald.Search_StreamSearchByIDServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamSearchByID(stream) +} + +func (s *server) MultiSearch(ctx context.Context, reqs *payload.Search_MultiRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiSearch(ctx, reqs) +} + +func (s *server) MultiSearchByID(ctx context.Context, reqs *payload.Search_MultiIDRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiSearchByID(ctx, reqs) +} + +func toSearchResponse(dists []model.Distance, err error) (res *payload.Search_Response, rerr error) { + if err != nil { + return nil, err + } + if len(dists) == 0 { + return nil, errors.ErrEmptySearchResult + } + res = new(payload.Search_Response) + res.Results = make([]*payload.Object_Distance, 0, len(dists)) + for _, dist := range dists { + res.Results = append(res.GetResults(), &payload.Object_Distance{ + Id: dist.ID, + Distance: dist.Distance, + }) + } + return res, nil +} diff --git a/pkg/agent/core/faiss/handler/grpc/search_test.go b/pkg/agent/core/faiss/handler/grpc/search_test.go new file mode 100644 index 00000000000..c239202905a --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/search_test.go @@ -0,0 +1,875 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_Search(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Search_Request +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Response +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Response, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.Search(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_SearchByID(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Search_IDRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Response +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Response, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.SearchByID(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamSearch(t *testing.T) { +// type args struct { +// stream vald.Search_StreamSearchServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamSearch(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamSearchByID(t *testing.T) { +// type args struct { +// stream vald.Search_StreamSearchByIDServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamSearchByID(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiSearch(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Search_MultiRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Responses +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Responses, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Responses, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiSearch(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiSearchByID(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Search_MultiIDRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Search_Responses +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Search_Responses, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Responses, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiSearchByID(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_toSearchResponse(t *testing.T) { +// type args struct { +// dists []model.Distance +// err error +// } +// type want struct { +// wantRes *payload.Search_Response +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, *payload.Search_Response, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Search_Response, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dists:nil, +// err:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dists:nil, +// err:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotRes, err := toSearchResponse(test.args.dists, test.args.err) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/update.go b/pkg/agent/core/faiss/handler/grpc/update.go new file mode 100644 index 00000000000..f8762eaea6e --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/update.go @@ -0,0 +1,173 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Update(ctx context.Context, req *payload.Update_Request) (res *payload.Object_Location, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.UpdateRPCName) + defer func() { + if span != nil { + span.End() + } + }() + + vec := req.GetVector() + if len(vec.GetVector()) != s.faiss.GetDimensionSize() { + err = errors.ErrIncompatibleDimensionSize(len(vec.GetVector()), int(s.faiss.GetDimensionSize())) + err = status.WrapWithInvalidArgument("Update API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: vec.GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + uuid := vec.GetId() + if len(uuid) == 0 { + err = errors.ErrInvalidUUID(uuid) + err = status.WrapWithInvalidArgument(fmt.Sprintf("Update API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + return nil, err + } + + err = s.faiss.UpdateWithTime(uuid, vec.GetVector(), req.GetConfig().GetTimestamp()) + if err != nil { + var attrs []attribute.KeyValue + if errors.Is(err, errors.ErrObjectIDNotFound(vec.GetId())) { + err = status.WrapWithNotFound(fmt.Sprintf("Update API uuid %s not found", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeNotFound(err.Error()) + } else if errors.Is(err, errors.ErrUUIDNotFound(0)) || errors.Is(err, errors.ErrInvalidDimensionSize(len(vec.GetVector()), s.faiss.GetDimensionSize())) { + err = status.WrapWithInvalidArgument(fmt.Sprintf("Update API invalid argument for uuid \"%s\" vec \"%v\" detected", vec.GetId(), vec.GetVector()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid or vector", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + } else if errors.Is(err, errors.ErrUUIDAlreadyExists(vec.GetId())) { + err = status.WrapWithAlreadyExists(fmt.Sprintf("Update API uuid %s's same data already exists", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeAlreadyExists(err.Error()) + } else { + err = status.WrapWithInternal("Update API failed", err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + return s.newLocation(vec.GetId()), nil +} + +func (s *server) StreamUpdate(stream vald.Update_StreamUpdateServer) (err error) { + return s.UnimplementedValdServer.UnimplementedUpdateServer.StreamUpdate(stream) +} + +func (s *server) MultiUpdate(ctx context.Context, reqs *payload.Update_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedUpdateServer.MultiUpdate(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/update_test.go b/pkg/agent/core/faiss/handler/grpc/update_test.go new file mode 100644 index 00000000000..a9fdf65b62f --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/update_test.go @@ -0,0 +1,399 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_Update(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Update_Request +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Location +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Location, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Location, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.Update(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamUpdate(t *testing.T) { +// type args struct { +// stream vald.Update_StreamUpdateServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamUpdate(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiUpdate(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Update_MultiRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiUpdate(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/grpc/upsert.go b/pkg/agent/core/faiss/handler/grpc/upsert.go new file mode 100644 index 00000000000..48227c4e98e --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/upsert.go @@ -0,0 +1,36 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" +) + +func (s *server) Upsert(ctx context.Context, req *payload.Upsert_Request) (loc *payload.Object_Location, err error) { + return s.UnimplementedValdServer.UnimplementedUpsertServer.Upsert(ctx, req) +} + +func (s *server) StreamUpsert(stream vald.Upsert_StreamUpsertServer) (err error) { + return s.UnimplementedValdServer.UnimplementedUpsertServer.StreamUpsert(stream) +} + +func (s *server) MultiUpsert(ctx context.Context, reqs *payload.Upsert_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedUpsertServer.MultiUpsert(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/upsert_test.go b/pkg/agent/core/faiss/handler/grpc/upsert_test.go new file mode 100644 index 00000000000..0f469a4eebf --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/upsert_test.go @@ -0,0 +1,399 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +// NOT IMPLEMENTED BELOW +// +// func Test_server_Upsert(t *testing.T) { +// type args struct { +// ctx context.Context +// req *payload.Upsert_Request +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantLoc *payload.Object_Location +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Location, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotLoc *payload.Object_Location, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotLoc, w.wantLoc) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotLoc, w.wantLoc) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// req:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotLoc, err := s.Upsert(test.args.ctx, test.args.req) +// if err := checkFunc(test.want, gotLoc, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_StreamUpsert(t *testing.T) { +// type args struct { +// stream vald.Upsert_StreamUpsertServer +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// stream:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// err := s.StreamUpsert(test.args.stream) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_server_MultiUpsert(t *testing.T) { +// type args struct { +// ctx context.Context +// reqs *payload.Upsert_MultiRequest +// } +// type fields struct { +// name string +// ip string +// faiss service.Faiss +// eg errgroup.Group +// streamConcurrency int +// UnimplementedAgentServer agent.UnimplementedAgentServer +// UnimplementedValdServer vald.UnimplementedValdServer +// } +// type want struct { +// wantRes *payload.Object_Locations +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Object_Locations, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Object_Locations, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// reqs:nil, +// }, +// fields: fields { +// name:"", +// ip:"", +// faiss:nil, +// eg:nil, +// streamConcurrency:0, +// UnimplementedAgentServer:nil, +// UnimplementedValdServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// name: test.fields.name, +// ip: test.fields.ip, +// faiss: test.fields.faiss, +// eg: test.fields.eg, +// streamConcurrency: test.fields.streamConcurrency, +// UnimplementedAgentServer: test.fields.UnimplementedAgentServer, +// UnimplementedValdServer: test.fields.UnimplementedValdServer, +// } +// +// gotRes, err := s.MultiUpsert(test.args.ctx, test.args.reqs) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/rest/handler.go b/pkg/agent/core/faiss/handler/rest/handler.go new file mode 100644 index 00000000000..89e4c20d560 --- /dev/null +++ b/pkg/agent/core/faiss/handler/rest/handler.go @@ -0,0 +1,175 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package rest provides rest api logic +package rest + +import ( + "net/http" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/net/http/dump" + "github.com/vdaas/vald/internal/net/http/json" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/grpc" +) + +type Handler interface { + Index(w http.ResponseWriter, r *http.Request) (int, error) + Exists(w http.ResponseWriter, r *http.Request) (int, error) + Search(w http.ResponseWriter, r *http.Request) (int, error) + SearchByID(w http.ResponseWriter, r *http.Request) (int, error) + LinearSearch(w http.ResponseWriter, r *http.Request) (int, error) + LinearSearchByID(w http.ResponseWriter, r *http.Request) (int, error) + Insert(w http.ResponseWriter, r *http.Request) (int, error) + MultiInsert(w http.ResponseWriter, r *http.Request) (int, error) + Update(w http.ResponseWriter, r *http.Request) (int, error) + MultiUpdate(w http.ResponseWriter, r *http.Request) (int, error) + Remove(w http.ResponseWriter, r *http.Request) (int, error) + MultiRemove(w http.ResponseWriter, r *http.Request) (int, error) + CreateIndex(w http.ResponseWriter, r *http.Request) (int, error) + SaveIndex(w http.ResponseWriter, r *http.Request) (int, error) + CreateAndSaveIndex(w http.ResponseWriter, r *http.Request) (int, error) + GetObject(w http.ResponseWriter, r *http.Request) (int, error) +} + +type handler struct { + agent grpc.Server +} + +func New(opts ...Option) Handler { + h := new(handler) + + for _, opt := range append(defaultOptions, opts...) { + opt(h) + } + return h +} + +func (h *handler) Index(w http.ResponseWriter, r *http.Request) (int, error) { + data := make(map[string]interface{}) + return json.Handler(w, r, &data, func() (interface{}, error) { + return dump.Request(nil, data, r) + }) +} + +func (h *handler) Search(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Search(r.Context(), req) + }) +} + +func (h *handler) SearchByID(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_IDRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.SearchByID(r.Context(), req) + }) +} + +func (h *handler) LinearSearch(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.LinearSearch(r.Context(), req) + }) +} + +func (h *handler) LinearSearchByID(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_IDRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.LinearSearchByID(r.Context(), req) + }) +} + +func (h *handler) Insert(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Insert_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Insert(r.Context(), req) + }) +} + +func (h *handler) MultiInsert(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Insert_MultiRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.MultiInsert(r.Context(), req) + }) +} + +func (h *handler) Update(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Update_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Update(r.Context(), req) + }) +} + +func (h *handler) MultiUpdate(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Update_MultiRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.MultiUpdate(r.Context(), req) + }) +} + +func (h *handler) Remove(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Remove_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Remove(r.Context(), req) + }) +} + +func (h *handler) MultiRemove(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Remove_MultiRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.MultiRemove(r.Context(), req) + }) +} + +func (h *handler) CreateIndex(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Control_CreateIndexRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.CreateIndex(r.Context(), req) + }) +} + +func (h *handler) SaveIndex(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Empty + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.SaveIndex(r.Context(), req) + }) +} + +func (h *handler) CreateAndSaveIndex(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Control_CreateIndexRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + _, err = h.agent.CreateIndex(r.Context(), req) + if err != nil { + return nil, err + } + return h.agent.SaveIndex(r.Context(), nil) + }) +} + +func (h *handler) GetObject(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Object_VectorRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.GetObject(r.Context(), req) + }) +} + +func (h *handler) Exists(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Object_ID + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Exists(r.Context(), req) + }) +} diff --git a/pkg/agent/core/faiss/handler/rest/handler_test.go b/pkg/agent/core/faiss/handler/rest/handler_test.go new file mode 100644 index 00000000000..731c8972a44 --- /dev/null +++ b/pkg/agent/core/faiss/handler/rest/handler_test.go @@ -0,0 +1,1798 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package rest + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// opts []Option +// } +// type want struct { +// want Handler +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Handler) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Handler) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := New(test.args.opts...) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_Index(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// want int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// got, err := h.Index(test.args.w, test.args.r) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_Search(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.Search(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_SearchByID(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.SearchByID(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_LinearSearch(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.LinearSearch(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_LinearSearchByID(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.LinearSearchByID(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_Insert(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.Insert(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_MultiInsert(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.MultiInsert(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_Update(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.Update(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_MultiUpdate(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.MultiUpdate(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_Remove(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.Remove(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_MultiRemove(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.MultiRemove(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_CreateIndex(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.CreateIndex(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_SaveIndex(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.SaveIndex(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_CreateAndSaveIndex(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.CreateAndSaveIndex(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_GetObject(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.GetObject(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_handler_Exists(t *testing.T) { +// type args struct { +// w http.ResponseWriter +// r *http.Request +// } +// type fields struct { +// agent grpc.Server +// } +// type want struct { +// wantCode int +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, int, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCode int, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCode, w.wantCode) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCode, w.wantCode) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// w:nil, +// r:nil, +// }, +// fields: fields { +// agent:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// h := &handler{ +// agent: test.fields.agent, +// } +// +// gotCode, err := h.Exists(test.args.w, test.args.r) +// if err := checkFunc(test.want, gotCode, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/handler/rest/option.go b/pkg/agent/core/faiss/handler/rest/option.go new file mode 100644 index 00000000000..9b3b41e7d64 --- /dev/null +++ b/pkg/agent/core/faiss/handler/rest/option.go @@ -0,0 +1,30 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package rest provides rest api logic +package rest + +import "github.com/vdaas/vald/pkg/agent/core/faiss/handler/grpc" + +type Option func(*handler) + +var defaultOptions = []Option{} + +func WithAgent(a grpc.Server) Option { + return func(h *handler) { + h.agent = a + } +} diff --git a/pkg/agent/core/faiss/handler/rest/option_test.go b/pkg/agent/core/faiss/handler/rest/option_test.go new file mode 100644 index 00000000000..b5f9d42523e --- /dev/null +++ b/pkg/agent/core/faiss/handler/rest/option_test.go @@ -0,0 +1,102 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package rest + +// NOT IMPLEMENTED BELOW +// +// func TestWithAgent(t *testing.T) { +// type args struct { +// a grpc.Server +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// a:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// a:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithAgent(test.args.a) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/model/faiss.go b/pkg/agent/core/faiss/model/faiss.go new file mode 100644 index 00000000000..5d751a83e38 --- /dev/null +++ b/pkg/agent/core/faiss/model/faiss.go @@ -0,0 +1,23 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package model defines object structure +package model + +type Distance struct { + ID string + Distance float32 +} diff --git a/pkg/agent/core/faiss/router/option.go b/pkg/agent/core/faiss/router/option.go new file mode 100644 index 00000000000..4db27948195 --- /dev/null +++ b/pkg/agent/core/faiss/router/option.go @@ -0,0 +1,51 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package router provides implementation of Go API for routing http Handler wrapped by rest.Func +package router + +import ( + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/rest" +) + +// Option represents the functional option for router. +type Option func(*router) + +var defaultOptions = []Option{ + WithTimeout("3s"), +} + +// WithHandler returns the option to set the handler for the router. +func WithHandler(h rest.Handler) Option { + return func(r *router) { + r.handler = h + } +} + +// WithTimeout returns the option to set the timeout for the router. +func WithTimeout(timeout string) Option { + return func(r *router) { + r.timeout = timeout + } +} + +// WithErrGroup returns the option to set the error group for the router. +func WithErrGroup(eg errgroup.Group) Option { + return func(r *router) { + r.eg = eg + } +} diff --git a/pkg/agent/core/faiss/router/option_test.go b/pkg/agent/core/faiss/router/option_test.go new file mode 100644 index 00000000000..55911f71221 --- /dev/null +++ b/pkg/agent/core/faiss/router/option_test.go @@ -0,0 +1,274 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package router + +// NOT IMPLEMENTED BELOW +// +// func TestWithHandler(t *testing.T) { +// type args struct { +// h rest.Handler +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// h:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// h:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithHandler(test.args.h) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithTimeout(t *testing.T) { +// type args struct { +// timeout string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// timeout:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// timeout:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithTimeout(test.args.timeout) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithErrGroup(t *testing.T) { +// type args struct { +// eg errgroup.Group +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// eg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// eg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithErrGroup(test.args.eg) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/router/router.go b/pkg/agent/core/faiss/router/router.go new file mode 100644 index 00000000000..4b3cb2cc83c --- /dev/null +++ b/pkg/agent/core/faiss/router/router.go @@ -0,0 +1,170 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package router provides implementation of Go API for routing http Handler wrapped by rest.Func +package router + +import ( + "net/http" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/net/http/middleware" + "github.com/vdaas/vald/internal/net/http/routing" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/rest" +) + +type router struct { + handler rest.Handler + eg errgroup.Group + timeout string +} + +// New returns REST route&method information from handler interface. +func New(opts ...Option) http.Handler { + r := new(router) + + for _, opt := range append(defaultOptions, opts...) { + opt(r) + } + + h := r.handler + + return routing.New( + routing.WithMiddleware( + middleware.NewTimeout( + middleware.WithTimeout(r.timeout), + middleware.WithErrorGroup(r.eg), + )), + routing.WithRoutes([]routing.Route{ + { + "Index", + []string{ + http.MethodGet, + }, + "/", + h.Index, + }, + { + "Search", + []string{ + http.MethodPost, + }, + "/search", + h.Search, + }, + { + "Search By ID", + []string{ + http.MethodPost, + }, + "/id/search", + h.SearchByID, + }, + { + "LinearSearch", + []string{ + http.MethodPost, + }, + "/linearsearch", + h.LinearSearch, + }, + { + "LinearSearch By ID", + []string{ + http.MethodPost, + }, + "/id/linearsearch", + h.LinearSearchByID, + }, + { + "Insert", + []string{ + http.MethodPost, + }, + "/insert", + h.Insert, + }, + { + "Multiple Insert", + []string{ + http.MethodPost, + }, + "/insert/multi", + h.MultiInsert, + }, + { + "Update", + []string{ + http.MethodPost, + http.MethodPatch, + http.MethodPut, + }, + "/update", + h.Update, + }, + { + "Multiple Update", + []string{ + http.MethodPost, + http.MethodPatch, + http.MethodPut, + }, + "/update/multi", + h.MultiUpdate, + }, + { + "Remove", + []string{ + http.MethodDelete, + }, + "/delete", + h.Remove, + }, + { + "Multiple Remove", + []string{ + http.MethodDelete, + http.MethodPost, + }, + "/delete/multi", + h.MultiRemove, + }, + { + "Create Index", + []string{ + http.MethodPost, + }, + "/index/create", + h.CreateIndex, + }, + { + "Save Index", + []string{ + http.MethodGet, + }, + "/index/save", + h.SaveIndex, + }, + { + "GetObject", + []string{ + http.MethodGet, + }, + "/object/{id}", + h.GetObject, + }, + }...)) +} diff --git a/pkg/agent/core/faiss/router/router_test.go b/pkg/agent/core/faiss/router/router_test.go new file mode 100644 index 00000000000..fb38bcc3cca --- /dev/null +++ b/pkg/agent/core/faiss/router/router_test.go @@ -0,0 +1,102 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package router + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// opts []Option +// } +// type want struct { +// want http.Handler +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, http.Handler) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got http.Handler) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := New(test.args.opts...) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/service/faiss.go b/pkg/agent/core/faiss/service/faiss.go new file mode 100644 index 00000000000..e81e63f9ee7 --- /dev/null +++ b/pkg/agent/core/faiss/service/faiss.go @@ -0,0 +1,1290 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package service manages the main logic of server. +package service + +import ( + "context" + "encoding/gob" + "io/fs" + "math" + "os" + "path/filepath" + "reflect" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/vdaas/vald/internal/config" + core "github.com/vdaas/vald/internal/core/algorithm/faiss" + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/pkg/agent/core/faiss/model" + "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" + "github.com/vdaas/vald/pkg/agent/core/ngt/service/vqueue" + "github.com/vdaas/vald/pkg/agent/internal/metadata" +) + +type ( + Faiss interface { + Start(ctx context.Context) <-chan error + Train(nb int, xb []float32) error + Insert(uuid string, xb []float32) error + InsertWithTime(uuid string, vec []float32, t int64) error + Update(uuid string, vec []float32) error + UpdateWithTime(uuid string, vec []float32, t int64) error + CreateIndex(ctx context.Context) error + SaveIndex(ctx context.Context) error + CreateAndSaveIndex(ctx context.Context) error + Search(k, nq uint32, xq []float32) ([]model.Distance, error) + Delete(uuid string) error + DeleteWithTime(uuid string, t int64) error + Exists(uuid string) (uint32, bool) + IsIndexing() bool + IsSaving() bool + NumberOfCreateIndexExecution() uint64 + NumberOfProactiveGCExecution() uint64 + Len() uint64 + InsertVQueueBufferLen() uint64 + DeleteVQueueBufferLen() uint64 + GetDimensionSize() int + GetTrainSize() int + Close(ctx context.Context) error + } + + faiss struct { + core core.Faiss + eg errgroup.Group + kvs kvs.BidiMap + fmu sync.Mutex + fmap map[string]int64 // failure map for index + vq vqueue.Queue + addVecs []float32 + addIds []int64 + isTrained bool + trainSize int + icnt uint64 + + // statuses + indexing atomic.Value + saving atomic.Value + cimu sync.Mutex // create index mutex + lastNocie uint64 // last number of create index execution this value prevent unnecessary saveindex + + // counters + nocie uint64 // number of create index execution + nogce uint64 // number of proactive GC execution + wfci uint64 // wait for create indexing + + // configurations + inMem bool // in-memory mode + dim int // dimension size + nlist int // the number of Voronoi cells + m int // number of subquantizers + alen int // auto indexing length + dur time.Duration // auto indexing check duration + sdur time.Duration // auto save index check duration + lim time.Duration // auto indexing time limit + minLit time.Duration // minimum load index timeout + maxLit time.Duration // maximum load index timeout + litFactor time.Duration // load index timeout factor + enableProactiveGC bool // if this value is true, agent component will purge GC memory more proactive + enableCopyOnWrite bool // if this value is true, agent component will write backup file using Copy on Write and saves old files to the old directory + path string // index path + smu sync.Mutex // save index lock + tmpPath atomic.Value // temporary index path for Copy on Write + oldPath string // old volume path + basePath string // index base directory for CoW + cowmu sync.Mutex // copy on write move lock + dcd bool // disable commit daemon + idelay time.Duration // initial delay duration + kvsdbConcurrency int // kvsdb concurrency + } +) + +const ( + kvsFileName = "faiss-meta.kvsdb" + kvsTimestampFileName = "faiss-timestamp.kvsdb" + noTimeStampFile = -1 + + oldIndexDirName = "backup" + originIndexDirName = "origin" + + // ref: https://github.com/facebookresearch/faiss/wiki/FAQ#can-i-ignore-warning-clustering-xxx-points-to-yyy-centroids + // ref: https://github.com/facebookresearch/faiss/blob/main/faiss/Clustering.cpp#L38 + minPointsPerCentroid int = 39 +) + +func New(cfg *config.Faiss, opts ...Option) (Faiss, error) { + var ( + f = &faiss{ + fmap: make(map[string]int64), + dim: cfg.Dimension, + nlist: cfg.Nlist, + m: cfg.M, + enableProactiveGC: cfg.EnableProactiveGC, + enableCopyOnWrite: cfg.EnableCopyOnWrite, + kvsdbConcurrency: cfg.KVSDB.Concurrency, + } + err error + ) + + for _, opt := range append(defaultOptions, opts...) { + if err := opt(f); err != nil { + return nil, errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + } + } + + if len(f.path) == 0 { + f.inMem = true + } + + if f.enableCopyOnWrite && !f.inMem && len(f.path) != 0 { + sep := string(os.PathSeparator) + f.path, err = filepath.Abs(strings.ReplaceAll(f.path, sep+sep, sep)) + if err != nil { + log.Warn(err) + } + + f.basePath = f.path + f.oldPath = file.Join(f.basePath, oldIndexDirName) + f.path = file.Join(f.basePath, originIndexDirName) + err = file.MkdirAll(f.oldPath, fs.ModePerm) + if err != nil { + log.Warn(err) + } + err = file.MkdirAll(f.path, fs.ModePerm) + if err != nil { + log.Warn(err) + } + err = f.mktmp() + if err != nil { + return nil, err + } + } + + err = f.initFaiss( + core.WithDimension(cfg.Dimension), + core.WithNlist(cfg.Nlist), + core.WithM(cfg.M), + core.WithNbitsPerIdx(cfg.NbitsPerIdx), + core.WithMetricType(cfg.MetricType), + ) + if err != nil { + return nil, err + } + + if f.dur == 0 || f.alen == 0 { + f.dcd = true + } + + if f.vq == nil { + f.vq, err = vqueue.New() + if err != nil { + return nil, err + } + } + + f.indexing.Store(false) + f.saving.Store(false) + + return f, nil +} + +func (f *faiss) initFaiss(opts ...core.Option) error { + var err error + + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + + if f.inMem { + log.Debug("vald agent starts with in-memory mode") + f.core, err = core.New(opts...) + return err + } + + ctx := context.Background() + err = f.load(ctx, f.path, opts...) + var current uint64 + if err != nil { + if !f.enableCopyOnWrite { + log.Debug("failed to load vald index from %s\t error: %v", f.path, err) + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } else if f.kvs.Len() > 0 { + f.kvs.Close() + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + + if f.core != nil { + f.core.Close() + f.core = nil + } + f.core, err = core.New(append(opts, core.WithIndexPath(f.path))...) + return err + } + + if errors.Is(err, errors.ErrIndicesAreTooFewComparedToMetadata) && f.kvs != nil { + current = f.kvs.Len() + log.Warnf( + "load vald primary index success from %s\t error: %v\tbut index data are too few %d compared to metadata count now trying to load from old copied index data from %s and compare them", + f.path, + err, + current, + f.oldPath, + ) + } else { + log.Warnf("failed to load vald primary index from %s\t error: %v\ttrying to load from old copied index data from %s", f.path, err, f.oldPath) + } + } else { + return nil + } + + err = f.load(ctx, f.oldPath, opts...) + if err == nil { + if current != 0 && f.kvs.Len() < current { + log.Warnf( + "load vald secondary index success from %s\t error: %v\tbut index data are too few %d compared to primary data now trying to load from primary index data again from %s and start up with them", + f.oldPath, + err, + f.kvs.Len(), + f.oldPath, + ) + + err = f.load(ctx, f.path, opts...) + if err == nil { + return nil + } + } else { + return nil + } + } + + log.Warnf("failed to load vald secondary index from %s and %s\t error: %v\ttrying to load from non-CoW index data from %s for backwards compatibility", f.path, f.oldPath, err, f.basePath) + err = f.load(ctx, f.basePath, opts...) + if err == nil { + file.CopyDir(ctx, f.basePath, f.path) + return nil + } + + tpath := f.tmpPath.Load().(string) + log.Warnf( + "failed to load vald backwards index from %s and %s and %s\t error: %v\tvald agent couldn't find any index from agent volume in %s trying to start as new index from %s", + f.path, + f.oldPath, + f.basePath, + err, + f.basePath, + tpath, + ) + + if f.core != nil { + f.core.Close() + f.core = nil + } + f.core, err = core.New(append(opts, core.WithIndexPath(tpath))...) + if err != nil { + return err + } + + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } else if f.kvs.Len() > 0 { + f.kvs.Close() + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + + return nil +} + +func (f *faiss) load(ctx context.Context, path string, opts ...core.Option) error { + exist, fi, err := file.ExistsWithDetail(path) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + err = errors.Wrapf(errors.ErrIndexFileNotFound, "index file does not exists,\tpath: %s,\terr: %v", path, err) + return err + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.Wrap(errors.ErrFailedToOpenFile(err, path, 0, fi.Mode()), "invalid permission for loading index path") + } + return err + case exist && fi != nil && fi.IsDir(): + if fi.Mode().IsDir() && !strings.HasSuffix(path, string(os.PathSeparator)) { + path += string(os.PathSeparator) + } + files, err := filepath.Glob(file.Join(filepath.Dir(path), "*")) + if err != nil || len(files) == 0 { + err = errors.Wrapf(errors.ErrIndexFileNotFound, "index path exists but no file does not exists in the directory,\tpath: %s,\tfiles: %v\terr: %v", path, files, err) + return err + } + if strings.HasSuffix(path, string(os.PathSeparator)) { + path = strings.TrimSuffix(path, string(os.PathSeparator)) + } + } + + metadataPath := file.Join(path, metadata.AgentMetadataFileName) + log.Debugf("index path: %s exists, now starting to check metadata from %s", path, metadataPath) + exist, fi, err = file.ExistsWithDetail(metadataPath) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + err = errors.Wrapf(errors.ErrIndexFileNotFound, "metadata file does not exists,\tpath: %s,\terr: %v", metadataPath, err) + return err + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.Wrap(errors.ErrFailedToOpenFile(err, metadataPath, 0, fi.Mode()), "invalid permission for loading metadata") + } + return err + } + + log.Debugf("index path: %s and metadata: %s exists, now starting to load metadata", path, metadataPath) + agentMetadata, err := metadata.Load(metadataPath) + if err != nil && errors.Is(err, fs.ErrNotExist) || agentMetadata == nil || agentMetadata.Faiss == nil || agentMetadata.Faiss.IndexCount == 0 { + err = errors.Wrapf(err, "cannot read metadata from path: %s\tmetadata: %s", path, agentMetadata) + return err + } + + kvsFilePath := file.Join(path, kvsFileName) + log.Debugf("index path: %s and metadata: %s exists and successfully load metadata, now starting to load kvs data from %s", path, metadataPath, kvsFilePath) + exist, fi, err = file.ExistsWithDetail(kvsFilePath) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + err = errors.Wrapf(errors.ErrIndexFileNotFound, "kvsdb file does not exists,\tpath: %s,\terr: %v", kvsFilePath, err) + return err + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.ErrFailedToOpenFile(err, kvsFilePath, 0, fi.Mode()) + } + err = errors.Wrapf(err, "invalid permission for loading kvsdb file from %s", kvsFilePath) + return err + } + + kvsTimestampFilePath := file.Join(path, kvsTimestampFileName) + log.Debugf("now starting to load kvs timestamp data from %s", kvsTimestampFilePath) + exist, fi, err = file.ExistsWithDetail(kvsTimestampFilePath) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + log.Warnf("timestamp kvsdb file does not exists,\tpath: %s,\terr: %v", kvsTimestampFilePath, err) + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.ErrFailedToOpenFile(err, kvsTimestampFilePath, 0, fi.Mode()) + } + log.Warnf("invalid permission for loading timestamp kvsdb file from %s", kvsTimestampFilePath) + } + + var timeout time.Duration + if agentMetadata != nil && agentMetadata.Faiss != nil { + log.Debugf("the backup index size is %d. starting to load...", agentMetadata.Faiss.IndexCount) + timeout = time.Duration( + math.Min( + math.Max( + float64(agentMetadata.Faiss.IndexCount)*float64(f.litFactor), + float64(f.minLit), + ), + float64(f.maxLit), + ), + ) + } else { + log.Debugf("cannot inspect the backup index size. starting to load default value.") + timeout = time.Duration(math.Min(float64(f.minLit), float64(f.maxLit))) + } + + log.Debugf( + "index path: %s and metadata: %s and kvsdb file: %s and timestamp kvsdb file: %s exists and successfully load metadata, now starting to load full index and kvs data in concurrent", + path, + metadataPath, + kvsFilePath, + kvsTimestampFilePath, + ) + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + eg, _ := errgroup.New(ctx) + eg.Go(safety.RecoverFunc(func() (err error) { + if f.core != nil { + f.core.Close() + f.core = nil + } + f.core, err = core.Load(append(opts, core.WithIndexPath(path))...) + if err != nil { + err = errors.Wrapf(err, "failed to load faiss index from path: %s", path) + return err + } + return nil + })) + + eg.Go(safety.RecoverFunc(func() (err error) { + err = f.loadKVS(ctx, path, timeout) + if err != nil { + err = errors.Wrapf(err, "failed to load kvsdb data from path: %s, %s", kvsFilePath, kvsTimestampFilePath) + return err + } + if f.kvs != nil && float64(agentMetadata.Faiss.IndexCount/2) > float64(f.kvs.Len()) { + return errors.ErrIndicesAreTooFewComparedToMetadata + } + return nil + })) + + ech := make(chan error, 1) + // NOTE: when it exceeds the timeout while loading, + // it should exit this function and leave this goroutine running. + f.eg.Go(safety.RecoverFunc(func() error { + defer close(ech) + ech <- safety.RecoverFunc(func() (err error) { + err = eg.Wait() + if err != nil { + log.Error(err) + return err + } + cancel() + return nil + })() + return nil + })) + + select { + case err := <-ech: + return err + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + log.Errorf("cannot load index backup data from %s within the timeout %s. the process is going to be killed.", path, timeout) + err := metadata.Store(metadataPath, + &metadata.Metadata{ + IsInvalid: true, + Faiss: &metadata.Faiss{ + IndexCount: 0, + }, + }, + ) + if err != nil { + return err + } + return errors.ErrIndexLoadTimeout + } + } + + return nil +} + +func (f *faiss) loadKVS(ctx context.Context, path string, timeout time.Duration) (err error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + eg, _ := errgroup.New(ctx) + + m := make(map[string]uint32) + mt := make(map[string]int64) + + eg.Go(safety.RecoverFunc(func() (err error) { + gob.Register(map[string]uint32{}) + var fi *os.File + fi, err = file.Open( + file.Join(path, kvsFileName), + os.O_RDONLY|os.O_SYNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if fi != nil { + derr := fi.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + err = gob.NewDecoder(fi).Decode(&m) + if err != nil { + log.Errorf("error decoding kvsdb file,\terr: %v", err) + return err + } + return nil + })) + + eg.Go(safety.RecoverFunc(func() (err error) { + gob.Register(map[string]int64{}) + var ft *os.File + ft, err = file.Open( + file.Join(path, kvsTimestampFileName), + os.O_RDONLY|os.O_SYNC, + fs.ModePerm, + ) + if err != nil { + log.Warnf("error opening timestamp kvsdb file,\terr: %v", err) + } + defer func() { + if ft != nil { + derr := ft.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + err = gob.NewDecoder(ft).Decode(&mt) + if err != nil { + log.Warnf("error decoding timestamp kvsdb file,\terr: %v", err) + } + return nil + })) + + err = eg.Wait() + if err != nil { + return err + } + + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } else if f.kvs.Len() > 0 { + f.kvs.Close() + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + for k, id := range m { + if ts, ok := mt[k]; ok { + f.kvs.Set(k, id, ts) + } else { + // NOTE: SaveIndex do not write ngt-timestamp.kvsdb with timestamp 0. + f.kvs.Set(k, id, 0) + f.fmap[k] = int64(id) + } + } + for k := range mt { + if _, ok := m[k]; !ok { + f.fmap[k] = noTimeStampFile + } + } + + return nil +} + +func (f *faiss) mktmp() error { + if !f.enableCopyOnWrite { + return nil + } + + path, err := file.MkdirTemp(file.Join(os.TempDir(), "vald")) + if err != nil { + log.Warnf("failed to create temporary index file path directory %s:\terr: %v", path, err) + return err + } + + f.tmpPath.Store(path) + + return nil +} + +func (f *faiss) Start(ctx context.Context) <-chan error { + if f.dcd { + return nil + } + + ech := make(chan error, 2) + f.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + if f.dur <= 0 { + f.dur = math.MaxInt64 + } + if f.sdur <= 0 { + f.sdur = math.MaxInt64 + } + if f.lim <= 0 { + f.lim = math.MaxInt64 + } + + if f.idelay > 0 { + timer := time.NewTimer(f.idelay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + timer.Stop() + } + + tick := time.NewTicker(f.dur) + sTick := time.NewTicker(f.sdur) + limit := time.NewTicker(f.lim) + defer tick.Stop() + defer sTick.Stop() + defer limit.Stop() + for { + err = nil + select { + case <-ctx.Done(): + err = f.CreateIndex(ctx) + if err != nil && !errors.Is(err, errors.ErrUncommittedIndexNotFound) { + ech <- err + return errors.Wrap(ctx.Err(), err.Error()) + } + return ctx.Err() + case <-tick.C: + if f.vq.IVQLen() >= f.alen { + err = f.CreateIndex(ctx) + } + case <-limit.C: + err = f.CreateAndSaveIndex(ctx) + case <-sTick.C: + err = f.SaveIndex(ctx) + } + if err != nil && err != errors.ErrUncommittedIndexNotFound { + ech <- err + runtime.Gosched() + err = nil + } + } + })) + + return ech +} + +func (f *faiss) Train(nb int, xb []float32) error { + err := f.core.Train(nb, xb) + if err != nil { + log.Errorf("failed to faiss train", err) + return err + } + + return nil +} + +func (f *faiss) Insert(uuid string, vec []float32) error { + return f.insert(uuid, vec, time.Now().UnixNano(), true) +} + +func (f *faiss) InsertWithTime(uuid string, vec []float32, t int64) error { + if t <= 0 { + t = time.Now().UnixNano() + } + + return f.insert(uuid, vec, t, true) +} + +func (f *faiss) insert(uuid string, xb []float32, t int64, validation bool) error { + if len(uuid) == 0 { + err := errors.ErrUUIDNotFound(0) + return err + } + + if validation { + _, ok := f.Exists(uuid) + if ok { + return errors.ErrUUIDAlreadyExists(uuid) + } + } + + return f.vq.PushInsert(uuid, xb, t) +} + +func (f *faiss) Update(uuid string, vec []float32) error { + return f.update(uuid, vec, time.Now().UnixNano()) +} + +func (f *faiss) UpdateWithTime(uuid string, vec []float32, t int64) error { + if t <= 0 { + t = time.Now().UnixNano() + } + return f.update(uuid, vec, t) +} + +func (f *faiss) update(uuid string, vec []float32, t int64) (err error) { + if err = f.readyForUpdate(uuid, vec); err != nil { + return err + } + + err = f.delete(uuid, t, true) // `true` is to return NotFound error with non-existent ID + if err != nil { + return err + } + + t++ + return f.insert(uuid, vec, t, false) +} + +func (f *faiss) readyForUpdate(uuid string, vec []float32) (err error) { + if len(uuid) == 0 { + return errors.ErrUUIDNotFound(0) + } + + if len(vec) != f.GetDimensionSize() { + return errors.ErrInvalidDimensionSize(len(vec), f.GetDimensionSize()) + } + + // not impl GetObject() + + return nil +} + +func (f *faiss) CreateIndex(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "vald/agent-faiss/service/Faiss.CreateIndex") + defer func() { + if span != nil { + span.End() + } + }() + + ic := f.vq.IVQLen() + f.vq.DVQLen() + (len(f.addVecs) / f.dim) + if ic == 0 { + return errors.ErrUncommittedIndexNotFound + } + + wf := atomic.AddUint64(&f.wfci, 1) + if wf > 1 { + atomic.AddUint64(&f.wfci, ^uint64(0)) + log.Debugf("concurrent create index waiting detected this request will be ignored, concurrent: %d", wf) + return nil + } + + err := func() error { + ticker := time.NewTicker(time.Millisecond * 100) + defer ticker.Stop() + // wait for not indexing & not saving + for f.IsIndexing() || f.IsSaving() { + runtime.Gosched() + select { + case <-ctx.Done(): + atomic.AddUint64(&f.wfci, ^uint64(0)) + return ctx.Err() + case <-ticker.C: + } + } + atomic.AddUint64(&f.wfci, ^uint64(0)) + return nil + }() + if err != nil { + return err + } + + f.cimu.Lock() + defer f.cimu.Unlock() + f.indexing.Store(true) + defer f.indexing.Store(false) + defer f.gc() + now := time.Now().UnixNano() + ic = f.vq.IVQLen() + f.vq.DVQLen() + (len(f.addVecs) / f.dim) + if ic == 0 { + return errors.ErrUncommittedIndexNotFound + } + + log.Infof("create index operation started, uncommitted indexes = %d", ic) + log.Debug("create index delete phase started") + f.vq.RangePopDelete(ctx, now, func(uuid string) bool { + log.Debugf("start delete operation for kvsdb id: %s", uuid) + oid, ok := f.kvs.Delete(uuid) + if !ok { + log.Warn(errors.ErrObjectIDNotFound(uuid)) + return true + } + log.Debugf("start remove operation for faiss index id: %s, oid: %d", uuid, oid) + ntotal, err := f.core.Remove(1, []int64{int64(oid)}) + if err != nil { + log.Errorf("failed to remove oid: %d from faiss index. error: %v", oid, err) + f.fmu.Lock() + f.fmap[uuid] = int64(oid) + f.fmu.Unlock() + } + log.Debugf("removed from faiss index and kvsdb id: %s, oid: %d, index size: %d", uuid, oid, ntotal) + return true + }) + log.Debug("create index delete phase finished") + + f.gc() + + log.Debug("create index insert phase started") + f.vq.RangePopInsert(ctx, now, func(uuid string, vector []float32, timestamp int64) bool { + log.Debugf("start stack operation for faiss index id: %s, icnt: %d", uuid, uint32(f.icnt)) + f.addVecs = append(f.addVecs, vector...) + f.addIds = append(f.addIds, int64(f.icnt)) + + log.Debugf("start insert operation for kvsdb id: %s, icnt: %d", uuid, uint32(f.icnt)) + f.kvs.Set(uuid, uint32(f.icnt), timestamp) + atomic.AddUint64(&f.icnt, 1) + + f.fmu.Lock() + _, ok := f.fmap[uuid] + if ok { + delete(f.fmap, uuid) + } + f.fmu.Unlock() + log.Debugf("finished to insert index and kvsdb id: %s, icnt: %d", uuid, uint32(f.icnt)) + return true + }) + + var max int + if f.nlist > int(math.Pow(2, float64(f.m))) { + max = f.nlist + } else { + max = int(math.Pow(2, float64(f.m))) + } + if !f.isTrained && len(f.addVecs)/f.dim >= max*minPointsPerCentroid { + log.Debug("faiss train phase started") + log.Debugf("max * minPointsPerCentroid: %d", max*minPointsPerCentroid) + err := f.core.Train(len(f.addVecs)/f.dim, f.addVecs) + if err != nil { + log.Errorf("failed to faiss train", err) + return err + } + f.isTrained = true + f.trainSize = len(f.addVecs) / f.dim + log.Debug("faiss train phase finished") + } + if f.isTrained && len(f.addVecs) > 0 { + log.Debug("faiss add phase started") + ntotal, err := f.core.Add(len(f.addVecs)/f.dim, f.addVecs, f.addIds) + if err != nil { + log.Errorf("failed to faiss add", err) + return err + } + f.addVecs = nil + f.addIds = nil + log.Debugf("is trained: %v, index size: %d", f.isTrained, ntotal) + log.Debug("faiss add phase finished") + } + log.Debug("create index insert phase finished") + + atomic.AddUint64(&f.nocie, 1) + log.Info("create index operation finished") + + return nil +} + +func (f *faiss) SaveIndex(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "vald/agent-faiss/service/Faiss.SaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + + if !f.inMem { + return f.saveIndex(ctx) + } + + return nil +} + +func (f *faiss) saveIndex(ctx context.Context) error { + nocie := atomic.LoadUint64(&f.nocie) + if atomic.LoadUint64(&f.lastNocie) == nocie || !f.isTrained { + return nil + } + atomic.SwapUint64(&f.lastNocie, nocie) + + err := func() error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + // wait for not indexing & not saving + for f.IsIndexing() || f.IsSaving() { + runtime.Gosched() + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + return nil + }() + if err != nil { + return err + } + + f.saving.Store(true) + defer f.gc() + defer f.saving.Store(false) + + // no cleanup invalid index + + eg, ectx := errgroup.New(ctx) + // we want to ensure the acutal kvs size between kvsdb and metadata, + // so we create this counter to count the actual kvs size instead of using kvs.Len() + var ( + kvsLen uint64 + path string + ) + + if f.enableCopyOnWrite { + path = f.tmpPath.Load().(string) + } else { + path = f.path + } + + f.smu.Lock() + defer f.smu.Unlock() + + eg.Go(safety.RecoverFunc(func() (err error) { + if f.kvs.Len() > 0 && path != "" { + m := make(map[string]uint32, f.Len()) + mt := make(map[string]int64, f.Len()) + var mu sync.Mutex + + f.kvs.Range(ectx, func(key string, id uint32, ts int64) bool { + mu.Lock() + m[key] = id + mt[key] = ts + mu.Unlock() + atomic.AddUint64(&kvsLen, 1) + return true + }) + + var fi *os.File + fi, err = file.Open( + file.Join(path, kvsFileName), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if fi != nil { + derr := fi.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + + gob.Register(map[string]uint32{}) + err = gob.NewEncoder(fi).Encode(&m) + if err != nil { + return err + } + + err = fi.Sync() + if err != nil { + return err + } + + m = make(map[string]uint32) + + var ft *os.File + ft, err = file.Open( + file.Join(path, kvsTimestampFileName), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if ft != nil { + derr := ft.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + + gob.Register(map[string]int64{}) + err = gob.NewEncoder(ft).Encode(&mt) + if err != nil { + return err + } + + err = ft.Sync() + if err != nil { + return err + } + + mt = make(map[string]int64) + } + + return nil + })) + + eg.Go(safety.RecoverFunc(func() (err error) { + f.fmu.Lock() + fl := len(f.fmap) + f.fmu.Unlock() + + if fl > 0 && path != "" { + var fi *os.File + fi, err = file.Open( + file.Join(path, "invalid-"+kvsFileName), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if fi != nil { + derr := fi.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + + gob.Register(map[string]int64{}) + f.fmu.Lock() + err = gob.NewEncoder(fi).Encode(&f.fmap) + f.fmu.Unlock() + if err != nil { + return err + } + + err = fi.Sync() + if err != nil { + return err + } + } + + return nil + })) + + eg.Go(safety.RecoverFunc(func() error { + return f.core.SaveIndexWithPath(path) + })) + + err = eg.Wait() + if err != nil { + return err + } + + err = metadata.Store( + file.Join(path, metadata.AgentMetadataFileName), + &metadata.Metadata{ + IsInvalid: false, + Faiss: &metadata.Faiss{ + IndexCount: kvsLen, + }, + }, + ) + if err != nil { + return err + } + + return f.moveAndSwitchSavedData(ctx) +} + +func (f *faiss) moveAndSwitchSavedData(ctx context.Context) error { + if !f.enableCopyOnWrite { + return nil + } + + var err error + f.cowmu.Lock() + defer f.cowmu.Unlock() + + err = file.MoveDir(ctx, f.path, f.oldPath) + if err != nil { + log.Warnf("failed to backup backup data from %s to %s error: %v", f.path, f.oldPath, err) + } + + path := f.tmpPath.Load().(string) + err = file.MoveDir(ctx, path, f.path) + if err != nil { + log.Warnf("failed to move temporary index data from %s to %s error: %v, trying to rollback secondary backup data from %s to %s", path, f.path, f.oldPath, f.path, err) + return file.MoveDir(ctx, f.oldPath, f.path) + } + defer log.Warnf("finished to copy index from %s => %s => %s", path, f.path, f.oldPath) + + return f.mktmp() +} + +func (f *faiss) CreateAndSaveIndex(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "vald/agent-faiss/service/Faiss.CreateAndSaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + + err := f.CreateIndex(ctx) + if err != nil && + !errors.Is(err, errors.ErrUncommittedIndexNotFound) && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + return err + } + + return f.SaveIndex(ctx) +} + +func (f *faiss) Search(k, nq uint32, xq []float32) ([]model.Distance, error) { + if f.IsIndexing() { + return nil, errors.ErrCreateIndexingIsInProgress + } + + sr, err := f.core.Search(int(k), int(nq), xq) + if err != nil { + if f.IsIndexing() { + return nil, errors.ErrCreateIndexingIsInProgress + } + + log.Errorf("cgo error detected: faiss api returned error %v", err) + return nil, err + } + + if len(sr) == 0 { + return nil, errors.ErrEmptySearchResult + } + + ds := make([]model.Distance, 0, len(sr)) + for _, d := range sr { + if err = d.Error; d.ID == 0 && err != nil { + log.Warnf("an error occurred while searching: %s", err) + continue + } + + key, _, ok := f.kvs.GetInverse(d.ID) + if ok { + ds = append(ds, model.Distance{ + ID: key, + Distance: d.Distance, + }) + } else { + log.Warn("not found", d.ID, d.Distance) + } + } + + return ds, nil +} + +func (f *faiss) Delete(uuid string) (err error) { + return f.delete(uuid, time.Now().UnixNano(), true) +} + +func (f *faiss) DeleteWithTime(uuid string, t int64) (err error) { + if t <= 0 { + t = time.Now().UnixNano() + } + + return f.delete(uuid, t, true) +} + +func (f *faiss) delete(uuid string, t int64, validation bool) error { + if len(uuid) == 0 { + return errors.ErrUUIDNotFound(0) + } + + if validation { + _, _, ok := f.kvs.Get(uuid) + if !ok && !f.vq.IVExists(uuid) { + return errors.ErrObjectIDNotFound(uuid) + } + } + + return f.vq.PushDelete(uuid, t) +} + +func (f *faiss) Exists(uuid string) (uint32, bool) { + var ( + oid uint32 + ok bool + ) + + ok = f.vq.IVExists(uuid) + if !ok { + oid, _, ok = f.kvs.Get(uuid) + if !ok { + log.Debugf("Exists\tuuid: %s's data not found in kvsdb and insert vqueue\terror: %v", uuid, errors.ErrObjectIDNotFound(uuid)) + return 0, false + } + if f.vq.DVExists(uuid) { + log.Debugf("Exists\tuuid: %s's data found in kvsdb and not found in insert vqueue, but delete vqueue data exists. the object will be delete soon\terror: %v", + uuid, errors.ErrObjectIDNotFound(uuid)) + return 0, false + } + } + + return oid, ok +} + +func (f *faiss) IsIndexing() bool { + i, ok := f.indexing.Load().(bool) + return i && ok +} + +func (f *faiss) IsSaving() bool { + s, ok := f.saving.Load().(bool) + return s && ok +} + +func (f *faiss) NumberOfCreateIndexExecution() uint64 { + return atomic.LoadUint64(&f.nocie) +} + +func (f *faiss) NumberOfProactiveGCExecution() uint64 { + return atomic.LoadUint64(&f.nogce) +} + +func (f *faiss) gc() { + if f.enableProactiveGC { + runtime.GC() + atomic.AddUint64(&f.nogce, 1) + } +} + +func (f *faiss) Len() uint64 { + return f.kvs.Len() +} + +func (f *faiss) InsertVQueueBufferLen() uint64 { + return uint64(f.vq.IVQLen()) +} + +func (f *faiss) DeleteVQueueBufferLen() uint64 { + return uint64(f.vq.DVQLen()) +} + +func (f *faiss) GetDimensionSize() int { + return f.dim +} + +func (f *faiss) GetTrainSize() int { + return f.trainSize +} + +func (f *faiss) Close(ctx context.Context) error { + err := f.kvs.Close() + if len(f.path) != 0 { + cerr := f.CreateIndex(ctx) + if cerr != nil && + !errors.Is(err, errors.ErrUncommittedIndexNotFound) && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + if err != nil { + err = errors.Wrap(cerr, err.Error()) + } else { + err = cerr + } + } + + serr := f.SaveIndex(ctx) + if serr != nil && + !errors.Is(err, errors.ErrUncommittedIndexNotFound) && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + if err != nil { + err = errors.Wrap(serr, err.Error()) + } else { + err = serr + } + } + } + + f.core.Close() + + return nil +} diff --git a/pkg/agent/core/faiss/service/faiss_test.go b/pkg/agent/core/faiss/service/faiss_test.go new file mode 100644 index 00000000000..8b3ba2d2d7c --- /dev/null +++ b/pkg/agent/core/faiss/service/faiss_test.go @@ -0,0 +1,8194 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// cfg *config.Faiss +// opts []Option +// } +// type want struct { +// want Faiss +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Faiss, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Faiss, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// cfg:nil, +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// cfg:nil, +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, err := New(test.args.cfg, test.args.opts...) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_initFaiss(t *testing.T) { +// type args struct { +// opts []core.Option +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// opts:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// opts:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.initFaiss(test.args.opts...) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_load(t *testing.T) { +// type args struct { +// ctx context.Context +// path string +// opts []core.Option +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// path:"", +// opts:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// path:"", +// opts:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.load(test.args.ctx, test.args.path, test.args.opts...) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_loadKVS(t *testing.T) { +// type args struct { +// ctx context.Context +// path string +// timeout time.Duration +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// path:"", +// timeout:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// path:"", +// timeout:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.loadKVS(test.args.ctx, test.args.path, test.args.timeout) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_mktmp(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.mktmp() +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Start(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want <-chan error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, <-chan error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got <-chan error) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.Start(test.args.ctx) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Train(t *testing.T) { +// type args struct { +// nb int +// xb []float32 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// nb:0, +// xb:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// nb:0, +// xb:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.Train(test.args.nb, test.args.xb) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Insert(t *testing.T) { +// type args struct { +// uuid string +// vec []float32 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// vec:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// vec:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.Insert(test.args.uuid, test.args.vec) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_InsertWithTime(t *testing.T) { +// type args struct { +// uuid string +// vec []float32 +// t int64 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// vec:nil, +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// vec:nil, +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.InsertWithTime(test.args.uuid, test.args.vec, test.args.t) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_insert(t *testing.T) { +// type args struct { +// uuid string +// xb []float32 +// t int64 +// validation bool +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// xb:nil, +// t:0, +// validation:false, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// xb:nil, +// t:0, +// validation:false, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.insert(test.args.uuid, test.args.xb, test.args.t, test.args.validation) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Update(t *testing.T) { +// type args struct { +// uuid string +// vec []float32 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// vec:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// vec:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.Update(test.args.uuid, test.args.vec) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_UpdateWithTime(t *testing.T) { +// type args struct { +// uuid string +// vec []float32 +// t int64 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// vec:nil, +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// vec:nil, +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.UpdateWithTime(test.args.uuid, test.args.vec, test.args.t) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_update(t *testing.T) { +// type args struct { +// uuid string +// vec []float32 +// t int64 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// vec:nil, +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// vec:nil, +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.update(test.args.uuid, test.args.vec, test.args.t) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_readyForUpdate(t *testing.T) { +// type args struct { +// uuid string +// vec []float32 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// vec:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// vec:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.readyForUpdate(test.args.uuid, test.args.vec) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_CreateIndex(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.CreateIndex(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_SaveIndex(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.SaveIndex(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_saveIndex(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.saveIndex(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_moveAndSwitchSavedData(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.moveAndSwitchSavedData(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_CreateAndSaveIndex(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.CreateAndSaveIndex(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Search(t *testing.T) { +// type args struct { +// k uint32 +// nq uint32 +// xq []float32 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want []model.Distance +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, []model.Distance, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got []model.Distance, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// k:0, +// nq:0, +// xq:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// k:0, +// nq:0, +// xq:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got, err := f.Search(test.args.k, test.args.nq, test.args.xq) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Delete(t *testing.T) { +// type args struct { +// uuid string +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.Delete(test.args.uuid) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_DeleteWithTime(t *testing.T) { +// type args struct { +// uuid string +// t int64 +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// t:0, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.DeleteWithTime(test.args.uuid, test.args.t) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_delete(t *testing.T) { +// type args struct { +// uuid string +// t int64 +// validation bool +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// t:0, +// validation:false, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// t:0, +// validation:false, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.delete(test.args.uuid, test.args.t, test.args.validation) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Exists(t *testing.T) { +// type args struct { +// uuid string +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want uint32 +// want1 bool +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, uint32, bool) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got uint32, got1 bool) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// if !reflect.DeepEqual(got1, w.want1) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got1, w.want1) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// uuid:"", +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// uuid:"", +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got, got1 := f.Exists(test.args.uuid) +// if err := checkFunc(test.want, got, got1); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_IsIndexing(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want bool +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, bool) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got bool) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.IsIndexing() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_IsSaving(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want bool +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, bool) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got bool) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.IsSaving() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_NumberOfCreateIndexExecution(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.NumberOfCreateIndexExecution() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_NumberOfProactiveGCExecution(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.NumberOfProactiveGCExecution() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_gc(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want) error { +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// f.gc() +// if err := checkFunc(test.want); err != nil { +// tt.Errorf("error = %v", err) +// } +// }) +// } +// } +// +// func Test_faiss_Len(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.Len() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_InsertVQueueBufferLen(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.InsertVQueueBufferLen() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_DeleteVQueueBufferLen(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.DeleteVQueueBufferLen() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_GetDimensionSize(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want int +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, int) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got int) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.GetDimensionSize() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_GetTrainSize(t *testing.T) { +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// want int +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, int) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got int) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// got := f.GetTrainSize() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_faiss_Close(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// core core.Faiss +// eg errgroup.Group +// kvs kvs.BidiMap +// fmap map[string]int64 +// vq vqueue.Queue +// addVecs []float32 +// addIds []int64 +// isTrained bool +// trainSize int +// icnt uint64 +// indexing atomic.Value +// saving atomic.Value +// lastNocie uint64 +// nocie uint64 +// nogce uint64 +// wfci uint64 +// inMem bool +// dim int +// nlist int +// m int +// alen int +// dur time.Duration +// sdur time.Duration +// lim time.Duration +// minLit time.Duration +// maxLit time.Duration +// litFactor time.Duration +// enableProactiveGC bool +// enableCopyOnWrite bool +// path string +// tmpPath atomic.Value +// oldPath string +// basePath string +// dcd bool +// idelay time.Duration +// kvsdbConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// core:nil, +// eg:nil, +// kvs:nil, +// fmap:nil, +// vq:nil, +// addVecs:nil, +// addIds:nil, +// isTrained:false, +// trainSize:0, +// icnt:0, +// indexing:nil, +// saving:nil, +// lastNocie:0, +// nocie:0, +// nogce:0, +// wfci:0, +// inMem:false, +// dim:0, +// nlist:0, +// m:0, +// alen:0, +// dur:nil, +// sdur:nil, +// lim:nil, +// minLit:nil, +// maxLit:nil, +// litFactor:nil, +// enableProactiveGC:false, +// enableCopyOnWrite:false, +// path:"", +// tmpPath:nil, +// oldPath:"", +// basePath:"", +// dcd:false, +// idelay:nil, +// kvsdbConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// f := &faiss{ +// core: test.fields.core, +// eg: test.fields.eg, +// kvs: test.fields.kvs, +// fmap: test.fields.fmap, +// vq: test.fields.vq, +// addVecs: test.fields.addVecs, +// addIds: test.fields.addIds, +// isTrained: test.fields.isTrained, +// trainSize: test.fields.trainSize, +// icnt: test.fields.icnt, +// indexing: test.fields.indexing, +// saving: test.fields.saving, +// lastNocie: test.fields.lastNocie, +// nocie: test.fields.nocie, +// nogce: test.fields.nogce, +// wfci: test.fields.wfci, +// inMem: test.fields.inMem, +// dim: test.fields.dim, +// nlist: test.fields.nlist, +// m: test.fields.m, +// alen: test.fields.alen, +// dur: test.fields.dur, +// sdur: test.fields.sdur, +// lim: test.fields.lim, +// minLit: test.fields.minLit, +// maxLit: test.fields.maxLit, +// litFactor: test.fields.litFactor, +// enableProactiveGC: test.fields.enableProactiveGC, +// enableCopyOnWrite: test.fields.enableCopyOnWrite, +// path: test.fields.path, +// tmpPath: test.fields.tmpPath, +// oldPath: test.fields.oldPath, +// basePath: test.fields.basePath, +// dcd: test.fields.dcd, +// idelay: test.fields.idelay, +// kvsdbConcurrency: test.fields.kvsdbConcurrency, +// } +// +// err := f.Close(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/service/option.go b/pkg/agent/core/faiss/service/option.go new file mode 100644 index 00000000000..1d271dffdc0 --- /dev/null +++ b/pkg/agent/core/faiss/service/option.go @@ -0,0 +1,271 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "math" + "math/big" + "os" + "time" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/rand" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/timeutil" +) + +// Option represent the functional option for faiss +type Option func(f *faiss) error + +var defaultOptions = []Option{ + WithErrGroup(errgroup.Get()), + WithAutoIndexCheckDuration("30m"), + WithAutoSaveIndexDuration("35m"), + WithAutoIndexDurationLimit("24h"), + WithAutoIndexLength(100), + WithInitialDelayMaxDuration("3m"), + WithMinLoadIndexTimeout("3m"), + WithMaxLoadIndexTimeout("10m"), + WithLoadIndexTimeoutFactor("1ms"), + WithProactiveGC(true), +} + +// WithErrGroup returns the functional option to set the error group. +func WithErrGroup(eg errgroup.Group) Option { + return func(f *faiss) error { + if eg != nil { + f.eg = eg + } + + return nil + } +} + +// WithEnableInMemoryMode returns the functional option to set the in memory mode flag. +func WithEnableInMemoryMode(enabled bool) Option { + return func(f *faiss) error { + f.inMem = enabled + + return nil + } +} + +// WithIndexPath returns the functional option to set the index path of the Faiss. +func WithIndexPath(path string) Option { + return func(f *faiss) error { + if path == "" { + return nil + } + f.path = file.Join(strings.TrimSuffix(path, string(os.PathSeparator))) + return nil + } +} + +// WithAutoIndexCheckDuration returns the functional option to set the index check duration. +func WithAutoIndexCheckDuration(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.dur = d + + return nil + } +} + +// WithAutoSaveIndexDuration returns the functional option to set the auto save index duration. +func WithAutoSaveIndexDuration(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.sdur = d + + return nil + } +} + +// WithAutoIndexDurationLimit returns the functional option to set the auto index duration limit. +func WithAutoIndexDurationLimit(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.lim = d + + return nil + } +} + +// WithAutoIndexLength returns the functional option to set the auto index length. +func WithAutoIndexLength(l int) Option { + return func(f *faiss) error { + f.alen = l + + return nil + } +} + +const ( + defaultDurationLimit float64 = 1.1 + defaultRandDuration int64 = 1 +) + +var ( + bigMaxFloat64 = big.NewFloat(math.MaxFloat64) + bigMinFloat64 = big.NewFloat(math.SmallestNonzeroFloat64) + bigMaxInt64 = big.NewInt(math.MaxInt64) + bigMinInt64 = big.NewInt(math.MinInt64) +) + +// WithInitialDelayMaxDuration returns the functional option to set the initial delay duration. +func WithInitialDelayMaxDuration(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + var dt time.Duration + switch { + case d <= time.Nanosecond: + return nil + case d <= time.Microsecond: + dt = time.Nanosecond + case d <= time.Millisecond: + dt = time.Microsecond + case d <= time.Second: + dt = time.Millisecond + default: + dt = time.Second + } + + dbs := math.Round(float64(d) / float64(dt)) + bdbs := big.NewFloat(dbs) + if dbs <= 0 || bigMaxFloat64.Cmp(bdbs) <= 0 || bigMinFloat64.Cmp(bdbs) >= 0 { + dbs = defaultDurationLimit + } + + rnd := int64(rand.LimitedUint32(uint64(dbs))) + brnd := big.NewInt(rnd) + if rnd <= 0 || bigMaxInt64.Cmp(brnd) <= 0 || bigMinInt64.Cmp(brnd) >= 0 { + rnd = defaultRandDuration + } + + delay := time.Duration(rnd) * dt + if delay <= 0 || delay >= math.MaxInt64 || delay <= math.MinInt64 { + return WithInitialDelayMaxDuration(dur)(f) + } + + f.idelay = delay + + return nil + } +} + +// WithMinLoadIndexTimeout returns the functional option to set the minimal load index timeout. +func WithMinLoadIndexTimeout(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.minLit = d + + return nil + } +} + +// WithMaxLoadIndexTimeout returns the functional option to set the maximum load index timeout. +func WithMaxLoadIndexTimeout(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.maxLit = d + + return nil + } +} + +// WithLoadIndexTimeoutFactor returns the functional option to set the factor of load index timeout. +func WithLoadIndexTimeoutFactor(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.litFactor = d + + return nil + } +} + +// WithProactiveGC returns the functional option to set the proactive GC enable flag. +func WithProactiveGC(enabled bool) Option { + return func(f *faiss) error { + f.enableProactiveGC = enabled + return nil + } +} + +// WithCopyOnWrite returns the functional option to set the CoW enable flag. +func WithCopyOnWrite(enabled bool) Option { + return func(f *faiss) error { + f.enableCopyOnWrite = enabled + return nil + } +} diff --git a/pkg/agent/core/faiss/service/option_test.go b/pkg/agent/core/faiss/service/option_test.go new file mode 100644 index 00000000000..06f897c3825 --- /dev/null +++ b/pkg/agent/core/faiss/service/option_test.go @@ -0,0 +1,1134 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +// NOT IMPLEMENTED BELOW +// +// func TestWithErrGroup(t *testing.T) { +// type args struct { +// eg errgroup.Group +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// eg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// eg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithErrGroup(test.args.eg) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithEnableInMemoryMode(t *testing.T) { +// type args struct { +// enabled bool +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// enabled:false, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// enabled:false, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithEnableInMemoryMode(test.args.enabled) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithIndexPath(t *testing.T) { +// type args struct { +// path string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithIndexPath(test.args.path) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithAutoIndexCheckDuration(t *testing.T) { +// type args struct { +// dur string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithAutoIndexCheckDuration(test.args.dur) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithAutoSaveIndexDuration(t *testing.T) { +// type args struct { +// dur string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithAutoSaveIndexDuration(test.args.dur) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithAutoIndexDurationLimit(t *testing.T) { +// type args struct { +// dur string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithAutoIndexDurationLimit(test.args.dur) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithAutoIndexLength(t *testing.T) { +// type args struct { +// l int +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// l:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// l:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithAutoIndexLength(test.args.l) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithInitialDelayMaxDuration(t *testing.T) { +// type args struct { +// dur string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithInitialDelayMaxDuration(test.args.dur) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithMinLoadIndexTimeout(t *testing.T) { +// type args struct { +// dur string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithMinLoadIndexTimeout(test.args.dur) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithMaxLoadIndexTimeout(t *testing.T) { +// type args struct { +// dur string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithMaxLoadIndexTimeout(test.args.dur) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithLoadIndexTimeoutFactor(t *testing.T) { +// type args struct { +// dur string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// dur:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithLoadIndexTimeoutFactor(test.args.dur) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithProactiveGC(t *testing.T) { +// type args struct { +// enabled bool +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// enabled:false, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// enabled:false, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithProactiveGC(test.args.enabled) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithCopyOnWrite(t *testing.T) { +// type args struct { +// enabled bool +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// enabled:false, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// enabled:false, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithCopyOnWrite(test.args.enabled) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/core/faiss/usecase/agentd.go b/pkg/agent/core/faiss/usecase/agentd.go new file mode 100644 index 00000000000..12dcc430ad8 --- /dev/null +++ b/pkg/agent/core/faiss/usecase/agentd.go @@ -0,0 +1,193 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package usecase + +import ( + "context" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + vald "github.com/vdaas/vald/apis/grpc/v1/vald" + iconf "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/observability" + faissmetrics "github.com/vdaas/vald/internal/observability/metrics/agent/core/faiss" + infometrics "github.com/vdaas/vald/internal/observability/metrics/info" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/pkg/agent/core/faiss/config" + handler "github.com/vdaas/vald/pkg/agent/core/faiss/handler/grpc" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/rest" + "github.com/vdaas/vald/pkg/agent/core/faiss/router" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" +) + +type run struct { + eg errgroup.Group + cfg *config.Data + faiss service.Faiss + server starter.Server + observability observability.Observability +} + +func New(cfg *config.Data) (r runner.Runner, err error) { + faiss, err := service.New( + cfg.Faiss, + service.WithErrGroup(errgroup.Get()), + service.WithEnableInMemoryMode(cfg.Faiss.EnableInMemoryMode), + service.WithIndexPath(cfg.Faiss.IndexPath), + service.WithAutoIndexCheckDuration(cfg.Faiss.AutoIndexCheckDuration), + service.WithAutoSaveIndexDuration(cfg.Faiss.AutoSaveIndexDuration), + service.WithAutoIndexDurationLimit(cfg.Faiss.AutoIndexDurationLimit), + service.WithAutoIndexLength(cfg.Faiss.AutoIndexLength), + service.WithInitialDelayMaxDuration(cfg.Faiss.InitialDelayMaxDuration), + service.WithMinLoadIndexTimeout(cfg.Faiss.MinLoadIndexTimeout), + service.WithMaxLoadIndexTimeout(cfg.Faiss.MaxLoadIndexTimeout), + service.WithLoadIndexTimeoutFactor(cfg.Faiss.LoadIndexTimeoutFactor), + service.WithProactiveGC(cfg.Faiss.EnableProactiveGC), + service.WithCopyOnWrite(cfg.Faiss.EnableCopyOnWrite), + ) + if err != nil { + return nil, err + } + + g, err := handler.New( + handler.WithFaiss(faiss), + handler.WithStreamConcurrency(cfg.Server.GetGRPCStreamConcurrency()), + ) + if err != nil { + return nil, err + } + + eg := errgroup.Get() + + grpcServerOptions := []server.Option{ + server.WithGRPCRegistFunc(func(srv *grpc.Server) { + agent.RegisterAgentServer(srv, g) + vald.RegisterValdServer(srv, g) + }), + server.WithPreStartFunc(func() error { + return nil + }), + server.WithPreStopFunction(func() error { + return nil + }), + } + + var obs observability.Observability + if cfg.Observability != nil && cfg.Observability.Enabled { + obs, err = observability.NewWithConfig( + cfg.Observability, + faissmetrics.New(faiss), + infometrics.New("agent_core_faiss_info", "Agent Faiss info", *cfg.Faiss), + ) + if err != nil { + return nil, err + } + } + + srv, err := starter.New( + starter.WithConfig(cfg.Server), + starter.WithREST(func(sc *iconf.Server) []server.Option { + return []server.Option{ + server.WithHTTPHandler( + router.New( + router.WithTimeout(sc.HTTP.HandlerTimeout), + router.WithErrGroup(eg), + router.WithHandler( + rest.New( + rest.WithAgent(g), + ), + ), + ), + ), + } + }), + starter.WithGRPC(func(sc *iconf.Server) []server.Option { + return grpcServerOptions + }), + ) + if err != nil { + return nil, err + } + + return &run{ + eg: eg, + faiss: faiss, + cfg: cfg, + server: srv, + observability: obs, + }, nil +} + +func (r *run) PreStart(ctx context.Context) error { + if r.observability != nil { + return r.observability.PreStart(ctx) + } + + return nil +} + +func (r *run) Start(ctx context.Context) (<-chan error, error) { + ech := make(chan error, 3) + var oech, nech, sech <-chan error + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + if r.observability != nil { + oech = r.observability.Start(ctx) + } + nech = r.faiss.Start(ctx) + sech = r.server.ListenAndServe(ctx) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-oech: + case err = <-nech: + case err = <-sech: + } + if err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + case ech <- err: + } + } + } + })) + + return ech, nil +} + +func (r *run) PreStop(ctx context.Context) error { + return nil +} + +func (r *run) Stop(ctx context.Context) error { + if r.observability != nil { + r.observability.Stop(ctx) + } + + return r.server.Shutdown(ctx) +} + +func (r *run) PostStop(ctx context.Context) error { + r.faiss.Close(ctx) + return nil +} diff --git a/pkg/agent/core/faiss/usecase/agentd_test.go b/pkg/agent/core/faiss/usecase/agentd_test.go new file mode 100644 index 00000000000..8d352eb6cdc --- /dev/null +++ b/pkg/agent/core/faiss/usecase/agentd_test.go @@ -0,0 +1,685 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// cfg *config.Data +// } +// type want struct { +// wantR runner.Runner +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, runner.Runner, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotR runner.Runner, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotR, w.wantR) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotR, w.wantR) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotR, err := New(test.args.cfg) +// if err := checkFunc(test.want, gotR, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_run_PreStart(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// eg errgroup.Group +// cfg *config.Data +// faiss service.Faiss +// server starter.Server +// observability observability.Observability +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// r := &run{ +// eg: test.fields.eg, +// cfg: test.fields.cfg, +// faiss: test.fields.faiss, +// server: test.fields.server, +// observability: test.fields.observability, +// } +// +// err := r.PreStart(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_run_Start(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// eg errgroup.Group +// cfg *config.Data +// faiss service.Faiss +// server starter.Server +// observability observability.Observability +// } +// type want struct { +// want <-chan error +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, <-chan error, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got <-chan error, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// r := &run{ +// eg: test.fields.eg, +// cfg: test.fields.cfg, +// faiss: test.fields.faiss, +// server: test.fields.server, +// observability: test.fields.observability, +// } +// +// got, err := r.Start(test.args.ctx) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_run_PreStop(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// eg errgroup.Group +// cfg *config.Data +// faiss service.Faiss +// server starter.Server +// observability observability.Observability +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// r := &run{ +// eg: test.fields.eg, +// cfg: test.fields.cfg, +// faiss: test.fields.faiss, +// server: test.fields.server, +// observability: test.fields.observability, +// } +// +// err := r.PreStop(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_run_Stop(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// eg errgroup.Group +// cfg *config.Data +// faiss service.Faiss +// server starter.Server +// observability observability.Observability +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// r := &run{ +// eg: test.fields.eg, +// cfg: test.fields.cfg, +// faiss: test.fields.faiss, +// server: test.fields.server, +// observability: test.fields.observability, +// } +// +// err := r.Stop(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_run_PostStop(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// eg errgroup.Group +// cfg *config.Data +// faiss service.Faiss +// server starter.Server +// observability observability.Observability +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// eg:nil, +// cfg:nil, +// faiss:nil, +// server:nil, +// observability:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// r := &run{ +// eg: test.fields.eg, +// cfg: test.fields.cfg, +// faiss: test.fields.faiss, +// server: test.fields.server, +// observability: test.fields.observability, +// } +// +// err := r.PostStop(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/agent/internal/metadata/metadata.go b/pkg/agent/internal/metadata/metadata.go index b4bd0226c32..b4ac3ec701b 100644 --- a/pkg/agent/internal/metadata/metadata.go +++ b/pkg/agent/internal/metadata/metadata.go @@ -32,14 +32,19 @@ const ( ) type Metadata struct { - IsInvalid bool `json:"is_invalid" yaml:"is_invalid"` - NGT *NGT `json:"ngt,omitempty" yaml:"ngt"` + IsInvalid bool `json:"is_invalid" yaml:"is_invalid"` + NGT *NGT `json:"ngt,omitempty" yaml:"ngt"` + Faiss *Faiss `json:"faiss,omitempty" yaml:"faiss"` } type NGT struct { IndexCount uint64 `json:"index_count" yaml:"index_count"` } +type Faiss struct { + IndexCount uint64 `json:"index_count" yaml:"index_count"` +} + func Load(path string) (meta *Metadata, err error) { var fi os.FileInfo exists, fi, err := file.ExistsWithDetail(path) diff --git a/tests/e2e/crud/crud_faiss_test.go b/tests/e2e/crud/crud_faiss_test.go new file mode 100644 index 00000000000..548738a61ab --- /dev/null +++ b/tests/e2e/crud/crud_faiss_test.go @@ -0,0 +1,331 @@ +//go:build e2e + +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// package crud provides e2e tests using ann-benchmarks datasets +package crud + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/tests/e2e/hdf5" + "github.com/vdaas/vald/tests/e2e/kubernetes/client" + "github.com/vdaas/vald/tests/e2e/kubernetes/portforward" + "github.com/vdaas/vald/tests/e2e/operation" +) + +var ( + host string + port int + ds *hdf5.Dataset + + insertNum int + searchNum int + searchByIDNum int + getObjectNum int + updateNum int + upsertNum int + removeNum int + + insertFrom int + searchFrom int + searchByIDFrom int + getObjectFrom int + updateFrom int + upsertFrom int + removeFrom int + + waitAfterInsertDuration time.Duration + + kubeClient client.Client + namespace string + + forwarder *portforward.Portforward +) + +func init() { + testing.Init() + + flag.StringVar(&host, "host", "localhost", "hostname") + flag.IntVar(&port, "port", 8081, "gRPC port") + + flag.IntVar(&insertNum, "insert-num", 10000, "number of id-vector pairs used for insert") + flag.IntVar(&searchNum, "search-num", 10000, "number of id-vector pairs used for search") + flag.IntVar(&updateNum, "update-num", 10000, "number of id-vector pairs used for update") + flag.IntVar(&removeNum, "remove-num", 10000, "number of id-vector pairs used for remove") + + flag.IntVar(&insertFrom, "insert-from", 0, "first index of id-vector pairs used for insert") + flag.IntVar(&searchFrom, "search-from", 0, "first index of id-vector pairs used for search") + flag.IntVar(&updateFrom, "update-from", 0, "first index of id-vector pairs used for update") + flag.IntVar(&removeFrom, "remove-from", 0, "first index of id-vector pairs used for remove") + + datasetName := flag.String("dataset", "fashion-mnist-784-euclidean.hdf5", "dataset") + waitAfterInsert := flag.String("wait-after-insert", "3m", "wait duration after inserting vectors") + + pf := flag.Bool("portforward", false, "enable port forwarding") + pfPodName := flag.String("portforward-pod-name", "vald-gateway-0", "pod name (only for port forward)") + pfPodPort := flag.Int("portforward-pod-port", port, "pod gRPC port (only for port forward)") + + kubeConfig := flag.String("kubeconfig", file.Join(os.Getenv("HOME"), ".kube", "config"), "kubeconfig path") + flag.StringVar(&namespace, "namespace", "default", "namespace") + + flag.Parse() + + var err error + if *pf { + kubeClient, err = client.New(*kubeConfig) + if err != nil { + panic(err) + } + + forwarder = kubeClient.Portforward(namespace, *pfPodName, port, *pfPodPort) + + err = forwarder.Start() + if err != nil { + panic(err) + } + } + + fmt.Printf("loading dataset: %s ", *datasetName) + ds, err = hdf5.HDF5ToDataset(*datasetName) + if err != nil { + panic(err) + } + fmt.Println("loading finished") + + waitAfterInsertDuration, err = time.ParseDuration(*waitAfterInsert) + if err != nil { + panic(err) + } +} + +func teardown() { + if forwarder != nil { + forwarder.Close() + } +} + +func sleep(t *testing.T, dur time.Duration) { + t.Logf("%v sleep for %s.", time.Now(), dur) + time.Sleep(dur) + t.Logf("%v sleep finished.", time.Now()) +} + +func TestE2EInsertOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Insert(t, ctx, operation.Dataset{ + Train: ds.Train[insertFrom : insertFrom+insertNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2ESearchOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Search(t, ctx, operation.Dataset{ + Test: ds.Test[searchFrom : searchFrom+searchNum], + Neighbors: ds.Neighbors[searchFrom : searchFrom+searchNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2EUpdateOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.UpdateWithParameters( + t, + ctx, + operation.Dataset{ + Train: ds.Train[updateFrom : updateFrom+updateNum], + }, + true, + 1, + func(t *testing.T, status int32, msg string) error { + t.Helper() + + if status != int32(codes.NotFound) { + return errors.Errorf("the returned status is not NotFound on Update #1: %s", err) + } + + t.Logf("received a NotFound error on #1: %s", msg) + + return nil + }, + func(t *testing.T, err error) error { + t.Helper() + + st, _, _ := status.ParseError(err, codes.Unknown, "") + if st.Code() != codes.NotFound { + return err + } + + return nil + }, + ) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2ERemoveOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Remove(t, ctx, operation.Dataset{ + Train: ds.Train[removeFrom : removeFrom+removeNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2EInsertAndSearch(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Insert(t, ctx, operation.Dataset{ + Train: ds.Train[insertFrom : insertFrom+insertNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + sleep(t, waitAfterInsertDuration) + + err = op.Search(t, ctx, operation.Dataset{ + Test: ds.Test[searchFrom : searchFrom+searchNum], + Neighbors: ds.Neighbors[searchFrom : searchFrom+searchNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2EStandardCRUD(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Insert(t, ctx, operation.Dataset{ + Train: ds.Train[insertFrom : insertFrom+insertNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + sleep(t, waitAfterInsertDuration) + + err = op.Search(t, ctx, operation.Dataset{ + Test: ds.Test[searchFrom : searchFrom+searchNum], + Neighbors: ds.Neighbors[searchFrom : searchFrom+searchNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Exists(t, ctx, "0") + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.UpdateWithParameters( + t, + ctx, + operation.Dataset{ + Train: ds.Train[updateFrom : updateFrom+updateNum], + }, + true, + 1, + func(t *testing.T, status int32, msg string) error { + t.Helper() + + if status != int32(codes.NotFound) { + return errors.Errorf("the returned status is not NotFound on Update #1: %s", err) + } + + t.Logf("received a NotFound error on #1: %s", msg) + + return nil + }, + func(t *testing.T, err error) error { + t.Helper() + + st, _, _ := status.ParseError(err, codes.Unknown, "") + if st.Code() != codes.NotFound { + return err + } + + return nil + }, + ) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Remove(t, ctx, operation.Dataset{ + Train: ds.Train[removeFrom : removeFrom+removeNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} diff --git a/versions/FAISS_VERSION b/versions/FAISS_VERSION new file mode 100644 index 00000000000..10c088013f8 --- /dev/null +++ b/versions/FAISS_VERSION @@ -0,0 +1 @@ +1.7.4 diff --git a/versions/HELM_VERSION b/versions/HELM_VERSION index 960df81ad61..0fe5e707da7 100644 --- a/versions/HELM_VERSION +++ b/versions/HELM_VERSION @@ -1 +1,5 @@ +<<<<<<< HEAD v3.13.2 +======= +v3.12.2 +>>>>>>> feature/agent/new-agents-helm-template-base diff --git a/versions/JAEGER_OPERATOR_VERSION b/versions/JAEGER_OPERATOR_VERSION index 132775c22c9..12930484725 100644 --- a/versions/JAEGER_OPERATOR_VERSION +++ b/versions/JAEGER_OPERATOR_VERSION @@ -1 +1,5 @@ +<<<<<<< HEAD 2.49.0 +======= +2.46.2 +>>>>>>> feature/agent/new-agents-helm-template-base diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index c1d4be12544..9b62267d23b 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1,5 @@ +<<<<<<< HEAD 55.3.0 +======= +48.1.1 +>>>>>>> feature/agent/new-agents-helm-template-base diff --git a/versions/YQ_VERSION b/versions/YQ_VERSION index ce34746a73e..19f20bfdf4f 100644 --- a/versions/YQ_VERSION +++ b/versions/YQ_VERSION @@ -1 +1,5 @@ +<<<<<<< HEAD v4.40.5 +======= +v4.34.2 +>>>>>>> feature/agent/new-agents-helm-template-base