diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml index 75e278d2c72..6144df238a9 100644 --- a/.github/actions/detect-docker-image-tags/action.yaml +++ b/.github/actions/detect-docker-image-tags/action.yaml @@ -24,6 +24,8 @@ inputs: description: "image names" required: false default: "vdaas/vald-agent-ngt \ + vdaas/vald-agent-qbg \ + vdaas/vald-agent-sidecar \ vdaas/vald-discoverer-k8s \ vdaas/vald-lb-gateway \ vdaas/vald-manager-index" @@ -43,6 +45,7 @@ runs: run: | declare -A m=( ["vdaas/vald-agent-ngt"]="agent.image.tag" + ["vdaas/vald-agent-qbg"]="agent.image.tag" ["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag" ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag" ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag" diff --git a/.github/actions/wait-for-docker-image/action.yaml b/.github/actions/wait-for-docker-image/action.yaml index 375e03caaeb..9322b0eb4e1 100644 --- a/.github/actions/wait-for-docker-image/action.yaml +++ b/.github/actions/wait-for-docker-image/action.yaml @@ -20,6 +20,8 @@ inputs: description: "image names" required: false default: "vdaas/vald-agent-ngt \ + vdaas/vald-agent-qbg \ + vdaas/vald-agent-sidecar \ vdaas/vald-discoverer-k8s \ vdaas/vald-lb-gateway \ vdaas/vald-manager-index" diff --git a/.github/workflows/dockers-agent-qbg-image.yml b/.github/workflows/dockers-agent-qbg-image.yml new file mode 100644 index 00000000000..e3ad805b021 --- /dev/null +++ b/.github/workflows/dockers-agent-qbg-image.yml @@ -0,0 +1,153 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: agent-qbg" +on: + push: + branches: + - main + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-agent-qbg-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/qbg/**" + - "cmd/agent/core/qbg/**" + - "dockers/agent/core/qbg/Dockerfile" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-agent-qbg-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/qbg/**" + - "cmd/agent/core/qbg/**" + - "dockers/agent/core/qbg/Dockerfile" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-agent-qbg-image.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/agent/core/qbg/**" + - "cmd/agent/core/qbg/**" + - "dockers/agent/core/qbg/Dockerfile" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + build: + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) }} + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: agent-qbg + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: agent-qbg image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index f6dd2c15810..65d8ed7218d 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -84,6 +84,39 @@ jobs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: "trivy-results.sarif" + agent-qbg: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/agent-qbg + imagename=`make docker/name/agent-qbg` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" agent-sidecar: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index fd593b1c6c0..8b35c899ea2 100755 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ hack/go.mod.default3 # for mac .DS_Store +.nvimlog # for nvim .nvimlog diff --git a/Makefile b/Makefile index 9d326895fa1..60a7cd207df 100644 --- a/Makefile +++ b/Makefile @@ -36,9 +36,6 @@ MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>" VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION) -NGT_VERSION := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VERSION) -NGT_REPO = github.com/yahoojapan/NGT - GOPROXY=direct GO_VERSION := $(eval GO_VERSION := $(shell cat versions/GO_VERSION))$(GO_VERSION) GOARCH := $(eval GOARCH := $(shell go env GOARCH))$(GOARCH) @@ -51,8 +48,10 @@ TEST_NOT_IMPL_PLACEHOLDER = NOT IMPLEMENTED BELOW TEMP_DIR := $(eval TEMP_DIR := $(shell mktemp -d))$(TEMP_DIR) -OPERATOR_SDK_VERSION := $(eval OPERATOR_SDK_VERSION := $(shell cat versions/OPERATOR_SDK_VERSION))$(OPERATOR_SDK_VERSION) +NGT_REPO = github.com/yahoojapan/NGT +BUF_VERSION := $(eval BUF_VERSION := $(shell cat versions/BUF_VERSION))$(BUF_VERSION) +FAISS_VERSION := $(eval FAISS_VERSION := $(shell cat versions/FAISS_VERSION))$(FAISS_VERSION) GOLANGCILINT_VERSION := $(eval GOLANGCILINT_VERSION := $(shell cat versions/GOLANGCILINT_VERSION))$(GOLANGCILINT_VERSION) HELM_DOCS_VERSION := $(eval HELM_DOCS_VERSION := $(shell cat versions/HELM_DOCS_VERSION))$(HELM_DOCS_VERSION) HELM_VERSION := $(eval HELM_VERSION := $(shell cat versions/HELM_VERSION))$(HELM_VERSION) @@ -60,6 +59,8 @@ JAEGER_OPERATOR_VERSION := $(eval JAEGER_OPERATOR_VERSION := $(shell cat versi KIND_VERSION := $(eval KIND_VERSION := $(shell cat versions/KIND_VERSION))$(KIND_VERSION) KUBECTL_VERSION := $(eval KUBECTL_VERSION := $(shell cat versions/KUBECTL_VERSION))$(KUBECTL_VERSION) KUBELINTER_VERSION := $(eval KUBELINTER_VERSION := $(shell cat versions/KUBELINTER_VERSION))$(KUBELINTER_VERSION) +NGT_VERSION := $(eval NGT_VERSION := $(shell cat versions/NGT_VERSION))$(NGT_VERSION) +OPERATOR_SDK_VERSION := $(eval OPERATOR_SDK_VERSION := $(shell cat versions/OPERATOR_SDK_VERSION))$(OPERATOR_SDK_VERSION) OTEL_OPERATOR_VERSION := $(eval OTEL_OPERATOR_VERSION := $(shell cat versions/OTEL_OPERATOR_VERSION))$(OTEL_OPERATOR_VERSION) PROMETHEUS_STACK_VERSION := $(eval PROMETHEUS_STACK_VERSION := $(shell cat versions/PROMETHEUS_STACK_VERSION))$(PROMETHEUS_STACK_VERSION) PROTOBUF_VERSION := $(eval PROTOBUF_VERSION := $(shell cat versions/PROTOBUF_VERSION))$(PROTOBUF_VERSION) @@ -67,7 +68,6 @@ REVIEWDOG_VERSION := $(eval REVIEWDOG_VERSION := $(shell cat versions/RE TELEPRESENCE_VERSION := $(eval TELEPRESENCE_VERSION := $(shell cat versions/TELEPRESENCE_VERSION))$(TELEPRESENCE_VERSION) VALDCLI_VERSION := $(eval VALDCLI_VERSION := $(shell cat versions/VALDCLI_VERSION))$(VALDCLI_VERSION) YQ_VERSION := $(eval YQ_VERSION := $(shell cat versions/YQ_VERSION))$(YQ_VERSION) -BUF_VERSION := $(eval BUF_VERSION := $(shell cat versions/BUF_VERSION))$(BUF_VERSION) OTEL_OPERATOR_RELEASE_NAME ?= opentelemetry-operator PROMETHEUS_RELEASE_NAME ?= prometheus @@ -184,29 +184,31 @@ PROTO_PATHS = \ # - internal/test/comparator # - internal/test/mock GO_SOURCES = $(eval GO_SOURCES := $(shell find \ - ./cmd \ - ./hack \ - ./internal \ - ./pkg \ - -not -path './cmd/cli/*' \ - -not -path './internal/core/algorithm/ngt/*' \ - -not -path './internal/compress/gob/*' \ - -not -path './internal/compress/gzip/*' \ - -not -path './internal/compress/lz4/*' \ - -not -path './internal/compress/zstd/*' \ - -not -path './internal/db/storage/blob/s3/sdk/s3/*' \ - -not -path './internal/db/rdb/mysql/dbr/*' \ - -not -path './internal/test/comparator/*' \ - -not -path './internal/test/mock/*' \ - -not -path './hack/benchmark/internal/client/ngtd/*' \ - -not -path './hack/benchmark/internal/starter/agent/*' \ - -not -path './hack/benchmark/internal/starter/external/*' \ - -not -path './hack/benchmark/internal/starter/gateway/*' \ - -not -path './hack/gorules/*' \ - -not -path './hack/license/*' \ - -not -path './hack/swagger/*' \ - -not -path './hack/tools/*' \ - -not -path './tests/*' \ + $(ROOTDIR)/cmd \ + $(ROOTDIR)/hack \ + $(ROOTDIR)/internal \ + $(ROOTDIR)/pkg \ + -not -path '$(ROOTDIR)/cmd/cli/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/ngt/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/qbg/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/faiss/*' \ + -not -path '$(ROOTDIR)/internal/compress/gob/*' \ + -not -path '$(ROOTDIR)/internal/compress/gzip/*' \ + -not -path '$(ROOTDIR)/internal/compress/lz4/*' \ + -not -path '$(ROOTDIR)/internal/compress/zstd/*' \ + -not -path '$(ROOTDIR)/internal/db/storage/blob/s3/sdk/s3/*' \ + -not -path '$(ROOTDIR)/internal/db/rdb/mysql/dbr/*' \ + -not -path '$(ROOTDIR)/internal/test/comparator/*' \ + -not -path '$(ROOTDIR)/internal/test/mock/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/client/ngtd/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/agent/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/external/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/gateway/*' \ + -not -path '$(ROOTDIR)/hack/gorules/*' \ + -not -path '$(ROOTDIR)/hack/license/*' \ + -not -path '$(ROOTDIR)/hack/swagger/*' \ + -not -path '$(ROOTDIR)/hack/tools/*' \ + -not -path '$(ROOTDIR)/tests/*' \ -type f \ -name '*.go' \ -not -regex '.*options?\.go' \ @@ -214,29 +216,31 @@ GO_SOURCES = $(eval GO_SOURCES := $(shell find \ -not -name '*_mock.go' \ -not -name 'doc.go'))$(GO_SOURCES) GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \ - ./cmd \ - ./hack \ - ./internal \ - ./pkg \ - -not -path './cmd/cli/*' \ - -not -path './internal/core/algorithm/ngt/*' \ - -not -path './internal/compress/gob/*' \ - -not -path './internal/compress/gzip/*' \ - -not -path './internal/compress/lz4/*' \ - -not -path './internal/compress/zstd/*' \ - -not -path './internal/db/storage/blob/s3/sdk/s3/*' \ - -not -path './internal/db/rdb/mysql/dbr/*' \ - -not -path './internal/test/comparator/*' \ - -not -path './internal/test/mock/*' \ - -not -path './hack/benchmark/internal/client/ngtd/*' \ - -not -path './hack/benchmark/internal/starter/agent/*' \ - -not -path './hack/benchmark/internal/starter/external/*' \ - -not -path './hack/benchmark/internal/starter/gateway/*' \ - -not -path './hack/gorules/*' \ - -not -path './hack/license/*' \ - -not -path './hack/swagger/*' \ - -not -path './hack/tools/*' \ - -not -path './tests/*' \ + $(ROOTDIR)/cmd \ + $(ROOTDIR)/hack \ + $(ROOTDIR)/internal \ + $(ROOTDIR)/pkg \ + -not -path '$(ROOTDIR)/cmd/cli/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/ngt/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/qbg/*' \ + -not -path '$(ROOTDIR)/internal/core/algorithm/faiss/*' \ + -not -path '$(ROOTDIR)/internal/compress/gob/*' \ + -not -path '$(ROOTDIR)/internal/compress/gzip/*' \ + -not -path '$(ROOTDIR)/internal/compress/lz4/*' \ + -not -path '$(ROOTDIR)/internal/compress/zstd/*' \ + -not -path '$(ROOTDIR)/internal/db/storage/blob/s3/sdk/s3/*' \ + -not -path '$(ROOTDIR)/internal/db/rdb/mysql/dbr/*' \ + -not -path '$(ROOTDIR)/internal/test/comparator/*' \ + -not -path '$(ROOTDIR)/internal/test/mock/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/client/ngtd/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/agent/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/external/*' \ + -not -path '$(ROOTDIR)/hack/benchmark/internal/starter/gateway/*' \ + -not -path '$(ROOTDIR)/hack/gorules/*' \ + -not -path '$(ROOTDIR)/hack/license/*' \ + -not -path '$(ROOTDIR)/hack/swagger/*' \ + -not -path '$(ROOTDIR)/hack/tools/*' \ + -not -path '$(ROOTDIR)/tests/*' \ -type f \ -regex '.*options?\.go' \ -not -name '*_test.go' \ @@ -244,7 +248,7 @@ GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \ -not -name 'doc.go'))$(GO_OPTION_SOURCES) GO_SOURCES_INTERNAL = $(eval GO_SOURCES_INTERNAL := $(shell find \ - ./internal \ + $(ROOTDIR)/internal \ -type f \ -name '*.go' \ -not -name '*_test.go' \ @@ -329,39 +333,40 @@ all: clean deps .PHONY: clean ## clean -clean: - rm -rf vendor - go clean -cache -modcache -testcache -i -r - mv ./apis/grpc/v1/vald/vald.go $(TEMP_DIR)/vald.go - mv ./apis/grpc/v1/agent/core/agent.go $(TEMP_DIR)/agent.go - mv ./apis/grpc/v1/payload/interface.go $(TEMP_DIR)/interface.go +clean: \ + clean-generated \ + proto/all \ + deps \ + format + +.PHONY: clean-generated +## clean generated files +clean-generated: + mv $(ROOTDIR)/apis/grpc/v1/vald/vald.go $(TEMP_DIR)/vald.go + mv $(ROOTDIR)/apis/grpc/v1/agent/core/agent.go $(TEMP_DIR)/agent.go + mv $(ROOTDIR)/apis/grpc/v1/payload/interface.go $(TEMP_DIR)/interface.go rm -rf \ - /go/pkg \ - ./*.log \ - ./*.svg \ - ./apis/docs \ - ./apis/swagger \ - ./apis/grpc \ - ./bench \ - ./pprof \ - ./libs \ - $(GOCACHE) \ - ./go.sum \ - ./go.mod - mkdir -p ./apis/grpc/v1/vald - mv $(TEMP_DIR)/vald.go ./apis/grpc/v1/vald/vald.go - mkdir -p ./apis/grpc/v1/agent/core - mv $(TEMP_DIR)/agent.go ./apis/grpc/v1/agent/core/agent.go - mkdir -p ./apis/grpc/v1/payload - mv $(TEMP_DIR)/interface.go ./apis/grpc/v1/payload/interface.go - cp ./hack/go.mod.default ./go.mod + $(ROOTDIR)/*.log \ + $(ROOTDIR)/*.svg \ + $(ROOTDIR)/apis/docs \ + $(ROOTDIR)/apis/swagger \ + $(ROOTDIR)/apis/grpc \ + $(ROOTDIR)/bench \ + $(ROOTDIR)/pprof \ + $(ROOTDIR)/libs + mkdir -p $(ROOTDIR)/apis/grpc/v1/vald + mv $(TEMP_DIR)/vald.go $(ROOTDIR)/apis/grpc/v1/vald/vald.go + mkdir -p $(ROOTDIR)/apis/grpc/v1/agent/core + mv $(TEMP_DIR)/agent.go $(ROOTDIR)/apis/grpc/v1/agent/core/agent.go + mkdir -p $(ROOTDIR)/apis/grpc/v1/payload + mv $(TEMP_DIR)/interface.go $(ROOTDIR)/apis/grpc/v1/payload/interface.go .PHONY: license ## add license to files license: GOPRIVATE=$(GOPRIVATE) \ MAINTAINER=$(MAINTAINER) \ - go run -mod=readonly hack/license/gen/main.go ./ + go run -mod=readonly hack/license/gen/main.go $(ROOTDIR)/ .PHONY: init ## initialize development environment @@ -383,12 +388,11 @@ tools/install: \ .PHONY: update ## update deps, license, and run golines, gofumpt, goimports update: \ - clean \ + clean-generated \ update/libs \ proto/all \ deps \ - format \ - go/deps + format .PHONY: format ## format go codes @@ -407,10 +411,10 @@ format/go: \ gofumpt/install \ strictgoimports/install \ goimports/install - find ./ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOPATH)/bin/golines -w -m $(GOLINES_MAX_WIDTH) - find ./ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOPATH)/bin/gofumpt -w - find ./ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOPATH)/bin/strictgoimports -w - find ./ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs $(GOPATH)/bin/goimports -w + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOPATH)/bin/golines -w -m $(GOLINES_MAX_WIDTH) + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOPATH)/bin/gofumpt -w + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*[^\.pb]\.go' -print | xargs $(GOPATH)/bin/strictgoimports -w + find $(ROOTDIR)/ -type d -name .git -prune -o -type f -regex '.*\.go' -print | xargs $(GOPATH)/bin/goimports -w .PHONY: format/go/test ## run golines, gofumpt, goimports for go test files @@ -440,7 +444,7 @@ format/md: \ "charts/**/*.md" \ "apis/**/*.md" \ "tests/**/*.md" \ - "./*.md" + "$(ROOTDIR)/*.md" .PHONY: format/json format/json: \ @@ -469,7 +473,8 @@ deps/install: \ strictgoimports/install \ goimports/install \ prettier/install \ - go/deps + go/deps \ + go/example/deps .PHONY: version ## print vald version @@ -526,6 +531,18 @@ ngt/install: /usr/local/include/NGT/Capi.h rm -rf $(TEMP_DIR)/NGT-$(NGT_VERSION) ldconfig +.PHONY: faiss/install +## install Faiss +faiss/install: /usr/local/lib/libfaiss.so +/usr/local/lib/libfaiss.so: + curl -LO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz + tar zxf v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/ + cd $(TEMP_DIR)/faiss-$(FAISS_VERSION) && \ + cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=ON -B build . && \ + make -C build -j faiss && \ + make -C build install + ldconfig + .PHONY: lint ## run lints lint: vet diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index 7c46a488a77..d34f91f89b8 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -49,7 +49,43 @@ cmd/agent/core/ngt/ngt: \ -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ -X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \ - -X '$(GOPKG)/internal/info.NGTVersion=$(NGT_VERSION)' \ + -X '$(GOPKG)/internal/info.AlgorithmInfo=NGT-$(NGT_VERSION)' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ + -buildid=" \ + -mod=readonly \ + -modcacherw \ + -a \ + -tags "cgo osusergo netgo static_build" \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + $@ -version + +cmd/agent/core/faiss/faiss: \ + faiss/install \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find ./cmd/agent/core/faiss -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find ./pkg/agent/core/faiss ./pkg/agent/core/ngt/service/kvs ./pkg/agent/core/ngt/service/vqueue ./pkg/agent/internal -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + CFLAGS="$(CFLAGS)" \ + CXXFLAGS="$(CXXFLAGS)" \ + CGO_ENABLED=1 \ + CGO_CXXFLAGS="-g -Ofast -march=native" \ + CGO_FFLAGS="-g -Ofast -march=native" \ + CGO_LDFLAGS="-g -Ofast -march=native" \ + GO111MODULE=on \ + GOPRIVATE=$(GOPRIVATE) \ + go build \ + --ldflags "-w -linkmode 'external' \ + -extldflags '-fPIC -pthread -fopenmp -std=gnu++20 -lstdc++ -lm -z relro -z now $(EXTLDFLAGS)' \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$${CGO_ENABLED}' \ + -X '$(GOPKG)/internal/info.FaissVersion=$(FAISS_VERSION)' \ -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ -buildid=" \ -mod=readonly \ diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index c9fc96612df..7d3d74a7342 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -18,6 +18,7 @@ ## update vald libraries including tools update/libs: \ update/chaos-mesh \ + update/faiss \ update/go \ update/golangci-lint \ update/helm \ @@ -25,8 +26,8 @@ update/libs: \ update/helm-operator \ update/jaeger-operator \ update/kind \ - update/kubectl \ update/kube-linter \ + update/kubectl \ update/ngt \ update/prometheus-stack \ update/protobuf \ @@ -142,6 +143,11 @@ update/kube-linter: update/ngt: curl --silent https://api.github.com/repos/yahoojapan/NGT/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/NGT_VERSION +.PHONY: update/faiss +## update facebookresearch/faiss version +update/faiss: + curl --silent https://api.github.com/repos/facebookresearch/faiss/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/FAISS_VERSION + .PHONY: update/reviewdog ## update reviewdog version update/reviewdog: diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index a8858a27a69..67ae8f8aedd 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -52,6 +52,18 @@ docker/build/agent-ngt: --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ --build-arg MAINTAINER=$(MAINTAINER) +.PHONY: docker/build/agent-faiss +## build agent-faiss image +docker/build/agent-faiss: + $(DOCKER) build \ + $(DOCKER_OPTS) \ + -f dockers/agent/core/faiss/Dockerfile \ + -t $(ORG)/vald-agent-faiss:$(TAG) . \ + --build-arg GO_VERSION=$(GO_VERSION) \ + --build-arg DISTROLESS_IMAGE=$(DISTROLESS_IMAGE) \ + --build-arg DISTROLESS_IMAGE_TAG=$(DISTROLESS_IMAGE_TAG) \ + --build-arg MAINTAINER=$(MAINTAINER) + .PHONY: docker/name/agent-sidecar docker/name/agent-sidecar: @echo "$(ORG)/$(AGENT_SIDECAR_IMAGE)" diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index 6fa678fa5fc..6a57b8b9172 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -19,6 +19,16 @@ e2e: $(call run-e2e-crud-test,-run TestE2EStandardCRUD) +.PHONY: e2e/faiss +## run e2e/faiss +e2e/faiss: + #$(call run-e2e-crud-faiss-test,-run TestE2EInsertOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2ESearchOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2EUpdateOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2ERemoveOnly) + #$(call run-e2e-crud-faiss-test,-run TestE2EInsertAndSearch) + $(call run-e2e-crud-faiss-test,-run TestE2EStandardCRUD) + .PHONY: e2e/skip ## run e2e with skip exists operation e2e/skip: diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index 099541741ea..d9e92332b43 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -116,6 +116,28 @@ define run-e2e-crud-test -kubeconfig=$(KUBECONFIG) endef +define run-e2e-crud-faiss-test + go test \ + -race \ + -mod=readonly \ + $1 \ + -v $(ROOTDIR)/tests/e2e/crud/crud_faiss_test.go \ + -tags "e2e" \ + -timeout $(E2E_TIMEOUT) \ + -host=$(E2E_BIND_HOST) \ + -port=$(E2E_BIND_PORT) \ + -dataset=$(ROOTDIR)/hack/benchmark/assets/dataset/$(E2E_DATASET_NAME).hdf5 \ + -insert-num=$(E2E_INSERT_COUNT) \ + -search-num=$(E2E_SEARCH_COUNT) \ + -update-num=$(E2E_UPDATE_COUNT) \ + -remove-num=$(E2E_REMOVE_COUNT) \ + -wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \ + -portforward=$(E2E_PORTFORWARD_ENABLED) \ + -portforward-pod-name=$(E2E_TARGET_POD_NAME) \ + -portforward-pod-port=$(E2E_TARGET_PORT) \ + -namespace=$(E2E_TARGET_NAMESPACE) +endef + define run-e2e-multi-crud-test GOPRIVATE=$(GOPRIVATE) \ go test \ @@ -231,3 +253,13 @@ define gen-go-option-test-sources fi; \ done endef + +define gen-crd-schema + @[ -f $(ROOTDIR)/charts/vald-helm-operator/crds/$1.yaml ] \ + && mv $(ROOTDIR)/charts/vald-helm-operator/crds/$1.yaml $(TEMP_DIR)/$1.yaml || true + GOPRIVATE=$(GOPRIVATE) \ + go run -mod=readonly $(ROOTDIR)/hack/helm/schema/crd/main.go \ + $(ROOTDIR)/charts/$2/values.yaml > $(TEMP_DIR)/$1-spec.yaml + $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ + $(TEMP_DIR)/$1.yaml $(TEMP_DIR)/$1-spec.yaml > $(ROOTDIR)/charts/vald-helm-operator/crds/$1.yaml +endef diff --git a/Makefile.d/helm.mk b/Makefile.d/helm.mk index 52e60bed149..13ff9ba87bc 100644 --- a/Makefile.d/helm.mk +++ b/Makefile.d/helm.mk @@ -104,20 +104,10 @@ $(BINDIR)/yq: ## generate OpenAPI v3 schema for ValdRelease helm/schema/crd/vald: \ yq/install - mv charts/vald-helm-operator/crds/valdrelease.yaml $(TEMP_DIR)/valdrelease.yaml - GOPRIVATE=$(GOPRIVATE) \ - go run -mod=readonly hack/helm/schema/crd/main.go \ - charts/vald/values.yaml > $(TEMP_DIR)/valdrelease-spec.yaml - $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ - $(TEMP_DIR)/valdrelease.yaml $(TEMP_DIR)/valdrelease-spec.yaml > charts/vald-helm-operator/crds/valdrelease.yaml + $(call gen-crd-schema,valdrelease,vald) .PHONY: helm/schema/crd/vald-helm-operator ## generate OpenAPI v3 schema for ValdHelmOperatorRelease helm/schema/crd/vald-helm-operator: \ yq/install - mv charts/vald-helm-operator/crds/valdhelmoperatorrelease.yaml $(TEMP_DIR)/valdhelmoperatorrelease.yaml - GOPRIVATE=$(GOPRIVATE) \ - go run -mod=readonly hack/helm/schema/crd/main.go \ - charts/vald-helm-operator/values.yaml > $(TEMP_DIR)/valdhelmoperatorrelease-spec.yaml - $(BINDIR)/yq eval-all 'select(fileIndex==0).spec.versions[0].schema.openAPIV3Schema.properties.spec = select(fileIndex==1).spec | select(fileIndex==0)' \ - $(TEMP_DIR)/valdhelmoperatorrelease.yaml $(TEMP_DIR)/valdhelmoperatorrelease-spec.yaml > charts/vald-helm-operator/crds/valdhelmoperatorrelease.yaml + $(call gen-crd-schema,valdhelmoperatorrelease,vald-helm-operator) diff --git a/Makefile.d/k3d.mk b/Makefile.d/k3d.mk index 6cd083436cc..7cc2bd8711b 100644 --- a/Makefile.d/k3d.mk +++ b/Makefile.d/k3d.mk @@ -30,10 +30,19 @@ $(BINDIR)/k3d: .PHONY: k3d/start ## start k3d (kubernetes in docker) cluster k3d/start: - $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) --image docker.io/rancher/k3s:latest -v "/lib/modules:/lib/modules" - # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) -v "/lib/modules:/lib/modules" --host-pid-mode=true + $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) \ + --agents $(K3D_NODES) \ + --image docker.io/rancher/k3s:latest \ + --host-pid-mode=true \ + --port 8081:80@loadbalancer \ + --k3s-arg "--disable=traefik@server:*" \ + -v "/lib/modules:/lib/modules" + # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) -v "/lib/modules:/lib/modules" # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) -p "8081:80@loadbalancer" --agents $(K3D_NODES) --k3s-arg '--disable=traefik@all' - export KUBECONFIG="$(shell sudo $(K3D_COMMAND) kubeconfig merge -o $(TEMP_DIR)/k3d_$(K3D_CLUSTER_NAME)_kubeconfig.yaml $(K3D_CLUSTER_NAME))" + # K3D_KUBECONFIG_PATH="$(TEMP_DIR)/k3d_$(K3D_CLUSTER_NAME)_kubeconfig.yaml" + # $(K3D_COMMAND) kubeconfig merge -o $(K3D_KUBECONFIG_PATH) $(K3D_CLUSTER_NAME) + # export KUBECONFIG="$(K3D_KUBECONFIG_PATH)" + kubectl wait -n kube-system --for=condition=ready pod -l k8s-app=metrics-server --timeout=600s kubectl apply -f https://projectcontour.io/quickstart/operator.yaml kubectl apply -f https://projectcontour.io/quickstart/contour-custom-resource.yaml diff --git a/charts/vald-helm-operator/crds/valdrelease.yaml b/charts/vald-helm-operator/crds/valdrelease.yaml index 9f135ab791b..4106e79335d 100644 --- a/charts/vald-helm-operator/crds/valdrelease.yaml +++ b/charts/vald-helm-operator/crds/valdrelease.yaml @@ -116,6 +116,12 @@ spec: items: type: object x-kubernetes-preserve-unknown-fields: true + algorithm: + type: string + enum: + - ngt + - qbg + - faiss annotations: type: object x-kubernetes-preserve-unknown-fields: true @@ -314,7 +320,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -732,6 +738,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -1222,7 +1230,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -1592,6 +1600,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2068,7 +2078,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -2429,6 +2439,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2879,7 +2891,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -3277,6 +3289,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -4240,7 +4254,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -4638,6 +4652,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -5399,7 +5415,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -5797,6 +5813,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -6536,7 +6554,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -6937,6 +6955,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: diff --git a/charts/vald/README.md b/charts/vald/README.md index a191e244d5b..fac30051fe0 100644 --- a/charts/vald/README.md +++ b/charts/vald/README.md @@ -50,8 +50,9 @@ Run the following command to install the chart, | agent.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms | list | `[]` | node affinity required node selectors | | agent.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity preferred scheduling terms | | agent.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod affinity required scheduling terms | -| agent.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["vald-agent-ngt"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | pod anti-affinity preferred scheduling terms | +| agent.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["vald-agent"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":100}]` | pod anti-affinity preferred scheduling terms | | agent.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | +| agent.algorithm | string | `"ngt"` | agent algorithm type. it should be `ngt` or `qbg` or `faiss`. | | agent.annotations | object | `{}` | deployment annotations | | agent.enabled | bool | `true` | agent enabled | | agent.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | @@ -67,7 +68,7 @@ Run the following command to install the chart, | agent.maxReplicas | int | `300` | maximum number of replicas. if HPA is disabled, this value will be ignored. | | agent.maxUnavailable | string | `"1"` | maximum number of unavailable replicas | | agent.minReplicas | int | `20` | minimum number of replicas. if HPA is disabled, the replicas will be set to this value | -| agent.name | string | `"vald-agent-ngt"` | name of agent deployment | +| agent.name | string | `"vald-agent"` | name of agent deployment | | agent.ngt.auto_create_index_pool_size | int | `10000` | batch process pool size of automatic create index operation | | agent.ngt.auto_index_check_duration | string | `"30m"` | check duration of automatic indexing | | agent.ngt.auto_index_duration_limit | string | `"24h"` | limit duration of automatic indexing | @@ -96,7 +97,7 @@ Run the following command to install the chart, | agent.ngt.vqueue.insert_buffer_pool_size | int | `10000` | insert slice pool buffer size | | agent.nodeName | string | `""` | node name | | agent.nodeSelector | object | `{}` | node selector | -| agent.observability | object | `{"otlp":{"attribute":{"service_name":"vald-agent-ngt"}}}` | observability config (overrides defaults.observability) | +| agent.observability | object | `{"otlp":{"attribute":{"service_name":"vald-agent"}}}` | observability config (overrides defaults.observability) | | agent.persistentVolume.accessMode | string | `"ReadWriteOncePod"` | agent pod storage accessMode | | agent.persistentVolume.enabled | bool | `false` | enables PVC. It is required to enable if agent pod's file store functionality is enabled with non in-memory mode | | agent.persistentVolume.mountPropagation | string | `"None"` | agent pod storage mountPropagation | @@ -304,7 +305,7 @@ Run the following command to install the chart, | defaults.observability.metrics.enable_goroutine | bool | `true` | goroutine metrics enabled | | defaults.observability.metrics.enable_memory | bool | `true` | memory metrics enabled | | defaults.observability.metrics.enable_version_info | bool | `true` | version info metrics enabled | -| defaults.observability.metrics.version_info_labels | list | `["vald_version","server_name","git_commit","build_time","go_version","go_os","go_arch","ngt_version"]` | enabled label names of version info | +| defaults.observability.metrics.version_info_labels | list | `["vald_version","server_name","git_commit","build_time","go_version","go_os","go_arch","algorithm_info"]` | enabled label names of version info | | defaults.observability.otlp.attribute | object | `{"namespace":"_MY_POD_NAMESPACE_","node_name":"_MY_NODE_NAME_","pod_name":"_MY_POD_NAME_","service_name":"vald"}` | default resource attribute | | defaults.observability.otlp.attribute.namespace | string | `"_MY_POD_NAMESPACE_"` | namespace | | defaults.observability.otlp.attribute.node_name | string | `"_MY_NODE_NAME_"` | node name | @@ -419,6 +420,7 @@ Run the following command to install the chart, | defaults.server_config.servers.grpc.port | int | `8081` | gRPC server port | | defaults.server_config.servers.grpc.server.grpc.bidirectional_stream_concurrency | int | `20` | gRPC server bidirectional stream concurrency | | defaults.server_config.servers.grpc.server.grpc.connection_timeout | string | `""` | gRPC server connection timeout | +| defaults.server_config.servers.grpc.server.grpc.enable_admin | bool | `true` | gRPC server admin option | | defaults.server_config.servers.grpc.server.grpc.enable_reflection | bool | `true` | gRPC server reflection option | | defaults.server_config.servers.grpc.server.grpc.header_table_size | int | `0` | gRPC server header table size | | defaults.server_config.servers.grpc.server.grpc.initial_conn_window_size | int | `2097152` | gRPC server initial connection window size | diff --git a/charts/vald/templates/agent/daemonset.yaml b/charts/vald/templates/agent/daemonset.yaml deleted file mode 100644 index 6a3f2f89cd2..00000000000 --- a/charts/vald/templates/agent/daemonset.yaml +++ /dev/null @@ -1,201 +0,0 @@ -# -# Copyright (C) 2019-2023 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -{{- $agent := .Values.agent -}} -{{- if and $agent.enabled (eq $agent.kind "DaemonSet") }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ $agent.name }} - labels: - app: {{ $agent.name }} - app.kubernetes.io/name: {{ include "vald.name" . }} - helm.sh/chart: {{ include "vald.chart" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Chart.Version }} - app.kubernetes.io/component: agent - {{- if $agent.annotations }} - annotations: - {{- toYaml $agent.annotations | nindent 4 }} - {{- end }} -spec: - revisionHistoryLimit: {{ $agent.revisionHistoryLimit }} - selector: - matchLabels: - app: {{ $agent.name }} - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: {{ $agent.rollingUpdate.maxUnavailable }} - maxSurge: {{ $agent.rollingUpdate.maxSurge }} - template: - metadata: - creationTimestamp: null - labels: - app: {{ $agent.name }} - app.kubernetes.io/name: {{ include "vald.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: agent - {{- $pprof := default .Values.defaults.server_config.metrics.pprof $agent.server_config.metrics.pprof }} - {{- if or $agent.podAnnotations $agent.sidecar.enabled $pprof.enabled }} - annotations: - {{- if $agent.podAnnotations }} - {{- toYaml $agent.podAnnotations | nindent 8 }} - {{- end }} - {{- if $pprof.enabled }} - pyroscope.io/scrape: "true" - pyroscope.io/application-name: {{ $agent.name }} - pyroscope.io/profile-cpu-enabled: "true" - pyroscope.io/profile-mem-enabled: "true" - pyroscope.io/port: "{{ $pprof.port }}" - {{- end }} - {{- if $agent.sidecar.enabled }} - checksum/configmap: {{ include (print $.Template.BasePath "/agent/configmap.yaml") . | sha256sum }} - checksum/sidecar-configmap: {{ include (print $.Template.BasePath "/agent/sidecar-configmap.yaml") . | sha256sum }} - {{- end }} - {{- end }} - spec: - {{- if or $agent.initContainers $agent.sidecar.initContainerEnabled }} - initContainers: - {{- if $agent.initContainers }} - {{- $initContainers := dict "initContainers" $agent.initContainers "Values" .Values "namespace" .Release.Namespace -}} - {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} - {{- end }} - {{- if $agent.sidecar.initContainerEnabled }} - - name: {{ $agent.sidecar.name }}-init - image: "{{ $agent.sidecar.image.repository }}:{{ default .Values.defaults.image.tag $agent.sidecar.image.tag }}" - imagePullPolicy: {{ $agent.sidecar.image.pullPolicy }} - {{- $servers := dict "Values" $agent.sidecar.server_config "default" .Values.defaults.server_config -}} - {{- include "vald.containerPorts" $servers | trim | nindent 10 }} - resources: - {{- toYaml $agent.sidecar.resources | nindent 12 }} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - {{- if $agent.securityContext }} - securityContext: - {{- toYaml $agent.securityContext | nindent 12 }} - {{- end }} - env: - - name: VALD_AGENT_SIDECAR_MODE - value: "initcontainer" - {{- if $agent.sidecar.env }} - {{- toYaml $agent.sidecar.env | nindent 12 }} - {{- end }} - volumeMounts: - - name: {{ $agent.sidecar.name }}-config - mountPath: /etc/server/ - {{- if $agent.volumeMounts }} - {{- toYaml $agent.volumeMounts | nindent 12 }} - {{- end }} - {{- end }} - {{- end }} - affinity: - {{- include "vald.affinity" $agent.affinity | nindent 8 }} - {{- if $agent.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml $agent.topologySpreadConstraints | nindent 8 }} - {{- end }} - containers: - - name: {{ $agent.name }} - image: "{{ $agent.image.repository }}:{{ default .Values.defaults.image.tag $agent.image.tag }}" - imagePullPolicy: {{ $agent.image.pullPolicy }} - {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config -}} - {{- include "vald.containerPorts" $servers | trim | nindent 10 }} - resources: - {{- toYaml $agent.resources | nindent 12 }} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - {{- if $agent.securityContext }} - securityContext: - {{- toYaml $agent.securityContext | nindent 12 }} - {{- end }} - {{- if $agent.env }} - env: - {{- toYaml $agent.env | nindent 12 }} - {{- end }} - volumeMounts: - - name: {{ $agent.name }}-config - mountPath: /etc/server/ - {{- if $agent.volumeMounts }} - {{- toYaml $agent.volumeMounts | nindent 12 }} - {{- end }} - {{- if $agent.sidecar.enabled }} - - name: {{ $agent.sidecar.name }} - image: "{{ $agent.sidecar.image.repository }}:{{ default .Values.defaults.image.tag $agent.sidecar.image.tag }}" - imagePullPolicy: {{ $agent.sidecar.image.pullPolicy }} - {{- $servers := dict "Values" $agent.sidecar.server_config "default" .Values.defaults.server_config -}} - {{- include "vald.containerPorts" $servers | trim | nindent 10 }} - resources: - {{- toYaml $agent.sidecar.resources | nindent 12 }} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - {{- if $agent.securityContext }} - securityContext: - {{- toYaml $agent.securityContext | nindent 12 }} - {{- end }} - env: - - name: VALD_AGENT_SIDECAR_MODE - value: "sidecar" - {{- if $agent.sidecar.env }} - {{- toYaml $agent.sidecar.env | nindent 12 }} - {{- end }} - volumeMounts: - - name: {{ $agent.sidecar.name }}-config - mountPath: /etc/server/ - {{- if $agent.volumeMounts }} - {{- toYaml $agent.volumeMounts | nindent 12 }} - {{- end }} - {{- end }} - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - {{- if $agent.podSecurityContext }} - securityContext: - {{- toYaml $agent.podSecurityContext | nindent 8 }} - {{- end }} - terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }} - volumes: - - name: {{ $agent.name }}-config - configMap: - defaultMode: 420 - name: {{ $agent.name }}-config - {{- if or $agent.sidecar.enabled $agent.sidecar.initContainerEnabled }} - - name: {{ $agent.sidecar.name }}-config - configMap: - defaultMode: 420 - name: {{ $agent.sidecar.name }}-config - {{- end }} - {{- if $agent.volumes }} - {{- toYaml $agent.volumes | nindent 8 }} - {{- end }} - {{- if $agent.nodeName }} - nodeName: {{ $agent.nodeName }} - {{- end }} - {{- if $agent.nodeSelector }} - nodeSelector: - {{- toYaml $agent.nodeSelector | nindent 8 }} - {{- end }} - {{- if $agent.tolerations }} - tolerations: - {{- toYaml $agent.tolerations | nindent 8 }} - {{- end }} - {{- if $agent.podPriority }} - {{- if $agent.podPriority.enabled }} - priorityClassName: {{ .Release.Namespace }}-{{ $agent.name }}-priority - {{- end }} - {{- end }} -status: -{{- end }} diff --git a/charts/vald/templates/agent/deployment.yaml b/charts/vald/templates/agent/deployment.yaml deleted file mode 100644 index d58c3a17962..00000000000 --- a/charts/vald/templates/agent/deployment.yaml +++ /dev/null @@ -1,205 +0,0 @@ -# -# Copyright (C) 2019-2023 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -{{- $agent := .Values.agent -}} -{{- if and $agent.enabled (eq $agent.kind "Deployment") }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $agent.name }} - labels: - app: {{ $agent.name }} - app.kubernetes.io/name: {{ include "vald.name" . }} - helm.sh/chart: {{ include "vald.chart" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Chart.Version }} - app.kubernetes.io/component: agent - {{- if $agent.annotations }} - annotations: - {{- toYaml $agent.annotations | nindent 4 }} - {{- end }} -spec: - progressDeadlineSeconds: {{ $agent.progressDeadlineSeconds }} - {{- if not $agent.hpa.enabled }} - replicas: {{ $agent.minReplicas }} - {{- end }} - revisionHistoryLimit: {{ $agent.revisionHistoryLimit }} - selector: - matchLabels: - app: {{ $agent.name }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: {{ $agent.rollingUpdate.maxSurge }} - maxUnavailable: {{ $agent.rollingUpdate.maxUnavailable }} - template: - metadata: - creationTimestamp: null - labels: - app: {{ $agent.name }} - app.kubernetes.io/name: {{ include "vald.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: agent - {{- $pprof := default .Values.defaults.server_config.metrics.pprof $agent.server_config.metrics.pprof }} - {{- if or $agent.podAnnotations $agent.sidecar.enabled $pprof.enabled }} - annotations: - {{- if $agent.podAnnotations }} - {{- toYaml $agent.podAnnotations | nindent 8 }} - {{- end }} - {{- if $pprof.enabled }} - pyroscope.io/scrape: "true" - pyroscope.io/application-name: {{ $agent.name }} - pyroscope.io/profile-cpu-enabled: "true" - pyroscope.io/profile-mem-enabled: "true" - pyroscope.io/port: "{{ $pprof.port }}" - {{- end }} - {{- if $agent.sidecar.enabled }} - checksum/configmap: {{ include (print $.Template.BasePath "/agent/configmap.yaml") . | sha256sum }} - checksum/sidecar-configmap: {{ include (print $.Template.BasePath "/agent/sidecar-configmap.yaml") . | sha256sum }} - {{- end }} - {{- end }} - spec: - {{- if or $agent.initContainers $agent.sidecar.initContainerEnabled }} - initContainers: - {{- if $agent.initContainers }} - {{- $initContainers := dict "initContainers" $agent.initContainers "Values" .Values "namespace" .Release.Namespace -}} - {{- include "vald.initContainers" $initContainers | trim | nindent 8 }} - {{- end }} - {{- if $agent.sidecar.initContainerEnabled }} - - name: {{ $agent.sidecar.name }}-init - image: "{{ $agent.sidecar.image.repository }}:{{ default .Values.defaults.image.tag $agent.sidecar.image.tag }}" - imagePullPolicy: {{ $agent.sidecar.image.pullPolicy }} - {{- $servers := dict "Values" $agent.sidecar.server_config "default" .Values.defaults.server_config -}} - {{- include "vald.containerPorts" $servers | trim | nindent 10 }} - resources: - {{- toYaml $agent.sidecar.resources | nindent 12 }} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - {{- if $agent.securityContext }} - securityContext: - {{- toYaml $agent.securityContext | nindent 12 }} - {{- end }} - env: - - name: VALD_AGENT_SIDECAR_MODE - value: "initcontainer" - {{- if $agent.sidecar.env }} - {{- toYaml $agent.sidecar.env | nindent 12 }} - {{- end }} - volumeMounts: - - name: {{ $agent.sidecar.name }}-config - mountPath: /etc/server/ - {{- if $agent.volumeMounts }} - {{- toYaml $agent.volumeMounts | nindent 12 }} - {{- end }} - {{- end }} - {{- end }} - affinity: - {{- include "vald.affinity" $agent.affinity | nindent 8 }} - {{- if $agent.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml $agent.topologySpreadConstraints | nindent 8 }} - {{- end }} - containers: - - name: {{ $agent.name }} - image: "{{ $agent.image.repository }}:{{ default .Values.defaults.image.tag $agent.image.tag }}" - imagePullPolicy: {{ $agent.image.pullPolicy }} - {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config -}} - {{- include "vald.containerPorts" $servers | trim | nindent 10 }} - resources: - {{- toYaml $agent.resources | nindent 12 }} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - {{- if $agent.securityContext }} - securityContext: - {{- toYaml $agent.securityContext | nindent 12 }} - {{- end }} - {{- if $agent.env }} - env: - {{- toYaml $agent.env | nindent 12 }} - {{- end }} - volumeMounts: - - name: {{ $agent.name }}-config - mountPath: /etc/server/ - {{- if $agent.volumeMounts }} - {{- toYaml $agent.volumeMounts | nindent 12 }} - {{- end }} - {{- if $agent.sidecar.enabled }} - - name: {{ $agent.sidecar.name }} - image: "{{ $agent.sidecar.image.repository }}:{{ default .Values.defaults.image.tag $agent.sidecar.image.tag }}" - imagePullPolicy: {{ $agent.sidecar.image.pullPolicy }} - {{- $servers := dict "Values" $agent.sidecar.server_config "default" .Values.defaults.server_config -}} - {{- include "vald.containerPorts" $servers | trim | nindent 10 }} - resources: - {{- toYaml $agent.sidecar.resources | nindent 12 }} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - {{- if $agent.securityContext }} - securityContext: - {{- toYaml $agent.securityContext | nindent 12 }} - {{- end }} - env: - - name: VALD_AGENT_SIDECAR_MODE - value: "sidecar" - {{- if $agent.sidecar.env }} - {{- toYaml $agent.sidecar.env | nindent 12 }} - {{- end }} - volumeMounts: - - name: {{ $agent.sidecar.name }}-config - mountPath: /etc/server/ - {{- if $agent.volumeMounts }} - {{- toYaml $agent.volumeMounts | nindent 12 }} - {{- end }} - {{- end }} - dnsPolicy: ClusterFirst - restartPolicy: Always - schedulerName: default-scheduler - {{- if $agent.podSecurityContext }} - securityContext: - {{- toYaml $agent.podSecurityContext | nindent 8 }} - {{- end }} - terminationGracePeriodSeconds: {{ $agent.terminationGracePeriodSeconds }} - volumes: - - name: {{ $agent.name }}-config - configMap: - defaultMode: 420 - name: {{ $agent.name }}-config - {{- if or $agent.sidecar.enabled $agent.sidecar.initContainerEnabled }} - - name: {{ $agent.sidecar.name }}-config - configMap: - defaultMode: 420 - name: {{ $agent.sidecar.name }}-config - {{- end }} - {{- if $agent.volumes }} - {{- toYaml $agent.volumes | nindent 8 }} - {{- end }} - {{- if $agent.nodeName }} - nodeName: {{ $agent.nodeName }} - {{- end }} - {{- if $agent.nodeSelector }} - nodeSelector: - {{- toYaml $agent.nodeSelector | nindent 8 }} - {{- end }} - {{- if $agent.tolerations }} - tolerations: - {{- toYaml $agent.tolerations | nindent 8 }} - {{- end }} - {{- if $agent.podPriority }} - {{- if $agent.podPriority.enabled }} - priorityClassName: {{ .Release.Namespace }}-{{ $agent.name }}-priority - {{- end }} - {{- end }} -status: -{{- end }} diff --git a/charts/vald/templates/agent/faiss/configmap.yaml b/charts/vald/templates/agent/faiss/configmap.yaml new file mode 100644 index 00000000000..53ff21f77d7 --- /dev/null +++ b/charts/vald/templates/agent/faiss/configmap.yaml @@ -0,0 +1,45 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $agent := .Values.agent -}} +{{- if and ($agent.enabled) (eq (lower $agent.algorithm) "faiss")}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $agent.name }}-config + labels: + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} + app.kubernetes.io/component: agent +data: + config.yaml: | + --- + version: {{ $agent.version }} + time_zone: {{ default .Values.defaults.time_zone $agent.time_zone }} + logging: + {{- $logging := dict "Values" $agent.logging "default" .Values.defaults.logging }} + {{- include "vald.logging" $logging | nindent 6 }} + server_config: + {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config }} + {{- include "vald.servers" $servers | nindent 6 }} + observability: + {{- $observability := dict "Values" $agent.observability "default" .Values.defaults.observability }} + {{- include "vald.observability" $observability | nindent 6 }} + faiss: + {{- toYaml $agent.faiss | nindent 6 }} +{{- end }} diff --git a/charts/vald/templates/agent/configmap.yaml b/charts/vald/templates/agent/ngt/configmap.yaml similarity index 96% rename from charts/vald/templates/agent/configmap.yaml rename to charts/vald/templates/agent/ngt/configmap.yaml index ced5a7701ac..3bc66e98f4b 100644 --- a/charts/vald/templates/agent/configmap.yaml +++ b/charts/vald/templates/agent/ngt/configmap.yaml @@ -14,7 +14,7 @@ # limitations under the License. # {{- $agent := .Values.agent -}} -{{- if $agent.enabled }} +{{- if and ($agent.enabled) (eq (lower $agent.algorithm) "ngt")}} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/vald/templates/agent/qbg/configmap.yaml b/charts/vald/templates/agent/qbg/configmap.yaml new file mode 100644 index 00000000000..d1350f7ccc3 --- /dev/null +++ b/charts/vald/templates/agent/qbg/configmap.yaml @@ -0,0 +1,45 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $agent := .Values.agent -}} +{{- if and ($agent.enabled) (eq (lower $agent.algorithm) "qbg")}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $agent.name }}-config + labels: + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} + app.kubernetes.io/component: agent +data: + config.yaml: | + --- + version: {{ $agent.version }} + time_zone: {{ default .Values.defaults.time_zone $agent.time_zone }} + logging: + {{- $logging := dict "Values" $agent.logging "default" .Values.defaults.logging }} + {{- include "vald.logging" $logging | nindent 6 }} + server_config: + {{- $servers := dict "Values" $agent.server_config "default" .Values.defaults.server_config }} + {{- include "vald.servers" $servers | nindent 6 }} + observability: + {{- $observability := dict "Values" $agent.observability "default" .Values.defaults.observability }} + {{- include "vald.observability" $observability | nindent 6 }} + qbg: + {{- toYaml $agent.qbg | nindent 6 }} +{{- end }} diff --git a/charts/vald/templates/agent/sidecar-configmap.yaml b/charts/vald/templates/agent/sidecar/configmap.yaml similarity index 100% rename from charts/vald/templates/agent/sidecar-configmap.yaml rename to charts/vald/templates/agent/sidecar/configmap.yaml diff --git a/charts/vald/templates/agent/sidecar-svc.yaml b/charts/vald/templates/agent/sidecar/svc.yaml similarity index 100% rename from charts/vald/templates/agent/sidecar-svc.yaml rename to charts/vald/templates/agent/sidecar/svc.yaml diff --git a/charts/vald/templates/agent/statefulset.yaml b/charts/vald/templates/agent/statefulset.yaml index c33245683e6..5da363220e7 100644 --- a/charts/vald/templates/agent/statefulset.yaml +++ b/charts/vald/templates/agent/statefulset.yaml @@ -101,6 +101,7 @@ spec: volumeMounts: - name: {{ $agent.sidecar.name }}-config mountPath: /etc/server/ + {{- if eq $agent.algorithm "ngt" }} {{- if not $agent.ngt.enable_in_memory_mode }} {{- if $agent.ngt.index_path }} {{- if $agent.persistentVolume.enabled }} @@ -113,6 +114,7 @@ spec: {{- end }} {{- end }} {{- end }} + {{- end }} {{- if $agent.volumeMounts }} {{- toYaml $agent.volumeMounts | nindent 12 }} {{- end }} diff --git a/charts/vald/values.schema.json b/charts/vald/values.schema.json index 6afe895d452..2df2c595fd8 100644 --- a/charts/vald/values.schema.json +++ b/charts/vald/values.schema.json @@ -61,6 +61,11 @@ } } }, + "algorithm": { + "type": "string", + "description": "agent algorithm type. it should be `ngt` or `qbg` or `faiss`.", + "enum": ["ngt", "qbg", "faiss"] + }, "annotations": { "type": "object", "description": "deployment annotations" @@ -336,7 +341,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -1065,6 +1070,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -1901,7 +1910,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -2555,6 +2564,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -3343,7 +3356,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -3983,6 +3996,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -4688,7 +4705,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -5383,6 +5400,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -6979,7 +7000,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -7680,6 +7701,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -8927,7 +8952,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -9628,6 +9653,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" @@ -10849,7 +10878,7 @@ "go_os", "go_arch", "cgo_enabled", - "ngt_version", + "algorithm_info", "build_cpu_info_flags" ] } @@ -11555,6 +11584,10 @@ "type": "string", "description": "gRPC server connection timeout" }, + "enable_admin": { + "type": "boolean", + "description": "gRPC server admin option" + }, "enable_reflection": { "type": "boolean", "description": "gRPC server reflection option" diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 45529107254..3047ee619a5 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -210,6 +210,9 @@ defaults: # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.enable_reflection", "type": "boolean"} # defaults.server_config.servers.grpc.server.grpc.enable_reflection -- gRPC server reflection option enable_reflection: true + # @schema {"name": "defaults.server_config.servers.grpc.server.grpc.enable_admin", "type": "boolean"} + # defaults.server_config.servers.grpc.server.grpc.enable_admin -- gRPC server admin option + enable_admin: true # @schema {"name": "defaults.server_config.servers.grpc.server.socket_option", "alias": "socket_option"} socket_option: # defaults.server_config.servers.grpc.server.socket_option.reuse_port -- server listen socket option for reuse_port functionality @@ -805,7 +808,7 @@ defaults: # @schema {"name": "defaults.observability.metrics.enable_version_info", "type": "boolean"} # defaults.observability.metrics.enable_version_info -- version info metrics enabled enable_version_info: true - # @schema {"name": "defaults.observability.metrics.version_info_labels", "type": "array", "items": {"type": "string", "enum": ["vald_version", "server_name", "git_commit", "build_time", "go_version", "go_os", "go_arch", "cgo_enabled", "ngt_version", "build_cpu_info_flags"]}} + # @schema {"name": "defaults.observability.metrics.version_info_labels", "type": "array", "items": {"type": "string", "enum": ["vald_version", "server_name", "git_commit", "build_time", "go_version", "go_os", "go_arch", "cgo_enabled", "algorithm_info", "build_cpu_info_flags"]}} # defaults.observability.metrics.version_info_labels -- enabled label names of version info version_info_labels: - "vald_version" @@ -815,7 +818,7 @@ defaults: - "go_version" - "go_os" - "go_arch" - - "ngt_version" + - "algorithm_info" # @schema {"name": "defaults.observability.metrics.enable_memory", "type": "boolean"} # defaults.observability.metrics.enable_memory -- memory metrics enabled enable_memory: true @@ -1420,6 +1423,10 @@ agent: # @schema {"name": "agent.version", "alias": "version"} # agent.version -- version of agent config version: v0.0.0 + # @schema {"name": "agent.algorithm", "type": "string", "enum": ["ngt", "qbg", "faiss"]} + # agent.algorithm -- agent algorithm type. + # it should be `ngt` or `qbg` or `faiss`. + algorithm: ngt # @schema {"name": "agent.time_zone", "type": "string"} # agent.time_zone -- Time zone time_zone: "" @@ -1428,7 +1435,7 @@ agent: logging: {} # @schema {"name": "agent.name", "type": "string"} # agent.name -- name of agent deployment - name: vald-agent-ngt + name: vald-agent # @schema {"name": "agent.kind", "type": "string", "enum": ["StatefulSet", "Deployment", "DaemonSet"]} # agent.kind -- deployment kind: Deployment, DaemonSet or StatefulSet kind: StatefulSet @@ -1599,7 +1606,7 @@ agent: - key: app operator: In values: - - vald-agent-ngt + - vald-agent # agent.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution -- pod anti-affinity required scheduling terms requiredDuringSchedulingIgnoredDuringExecution: [] # @schema {"name": "agent.topologySpreadConstraints", "alias": "topologySpreadConstraints"} @@ -1625,7 +1632,7 @@ agent: observability: otlp: attribute: - service_name: vald-agent-ngt + service_name: vald-agent # @schema {"name": "agent.resources", "alias": "resources"} # agent.resources -- compute resources. # recommended setting of memory requests = cluster memory * 0.4 / number of agent pods diff --git a/charts/vald/values/dev.yaml b/charts/vald/values/dev.yaml index c488ecb96ad..7562d676b8f 100644 --- a/charts/vald/values/dev.yaml +++ b/charts/vald/values/dev.yaml @@ -28,6 +28,8 @@ gateway: profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: "vald-lb-gateway" + ingress: + enabled: true resources: requests: cpu: 100m diff --git a/cmd/agent/core/faiss/main.go b/cmd/agent/core/faiss/main.go new file mode 100644 index 00000000000..d2aba32de10 --- /dev/null +++ b/cmd/agent/core/faiss/main.go @@ -0,0 +1,59 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package main provides program main +package main + +import ( + "context" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/pkg/agent/core/faiss/config" + "github.com/vdaas/vald/pkg/agent/core/faiss/usecase" +) + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "agent faiss" +) + +func main() { + if err := safety.RecoverFunc(func() error { + return runner.Do( + context.Background(), + runner.WithName(name), + runner.WithVersion(info.Version, maxVersion, minVersion), + runner.WithConfigLoader(func(path string) (interface{}, *config.GlobalConfig, error) { + cfg, err := config.NewConfig(path) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") + } + return cfg, &cfg.GlobalConfig, nil + }), + runner.WithDaemonInitializer(func(cfg interface{}) (runner.Runner, error) { + return usecase.New(cfg.(*config.Data)) + }), + ) + })(); err != nil { + log.Fatal(err, info.Get()) + return + } +} diff --git a/cmd/agent/core/faiss/sample.yaml b/cmd/agent/core/faiss/sample.yaml new file mode 100644 index 00000000000..b146729db73 --- /dev/null +++ b/cmd/agent/core/faiss/sample.yaml @@ -0,0 +1,123 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +version: v0.0.0 +time_zone: JST +logging: + format: raw + level: debug + logger: glg +server_config: + servers: + - name: grpc + host: 0.0.0.0 + port: 8081 + grpc: + bidirectional_stream_concurrency: 20 + connection_timeout: "" + header_table_size: 0 + initial_conn_window_size: 0 + initial_window_size: 0 + interceptors: [] + keepalive: + max_conn_age: "" + max_conn_age_grace: "" + max_conn_idle: "" + time: "" + timeout: "" + max_header_list_size: 0 + max_receive_message_size: 0 + max_send_message_size: 0 + read_buffer_size: 0 + write_buffer_size: 0 + mode: GRPC + probe_wait_time: 3s + restart: true + health_check_servers: + - name: readiness + host: 0.0.0.0 + port: 3001 + http: + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + shutdown_duration: 0s + write_timeout: "" + mode: "" + probe_wait_time: 3s + metrics_servers: + startup_strategy: + - grpc + - readiness + full_shutdown_duration: 600s + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +observability: + enabled: false + collector: + duration: 5s + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - faiss_version + trace: + enabled: false + sampling_rate: 1 + prometheus: + enabled: false + endpoint: /metrics + namespace: vald + jaeger: + enabled: false + collector_endpoint: "" + agent_endpoint: "jaeger-agent.default.svc.cluster.local:6831" + username: "" + password: "" + service_name: "vald-agent-faiss" + buffer_max_count: 10 +faiss: + auto_index_check_duration: 30m + auto_index_duration_limit: 24h + auto_index_length: 100 + auto_save_index_duration: 35m + dimension: 64 + enable_copy_on_write: false + enable_in_memory_mode: true + enable_proactive_gc: true + index_path: "" + initial_delay_max_duration: 3m + load_index_timeout_factor: 1ms + m: 8 # dimension % m == 0, train size >= 2^m(or nlist) * minPointsPerCentroid + max_load_index_timeout: 10m + metric_type: "inner_product" + min_load_index_timeout: 3m + nbits_per_idx: 8 + nlist: 100 diff --git a/cmd/agent/core/ngt/sample-cow.yaml b/cmd/agent/core/ngt/sample-cow.yaml index 0ad77413e23..bf52a7cdf71 100644 --- a/cmd/agent/core/ngt/sample-cow.yaml +++ b/cmd/agent/core/ngt/sample-cow.yaml @@ -87,7 +87,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/cmd/agent/core/ngt/sample.yaml b/cmd/agent/core/ngt/sample.yaml index 403d786b7a9..9ba4ef41f33 100644 --- a/cmd/agent/core/ngt/sample.yaml +++ b/cmd/agent/core/ngt/sample.yaml @@ -87,7 +87,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/cmd/discoverer/k8s/sample.yaml b/cmd/discoverer/k8s/sample.yaml index da5751cf85c..9c698b09f1c 100644 --- a/cmd/discoverer/k8s/sample.yaml +++ b/cmd/discoverer/k8s/sample.yaml @@ -100,7 +100,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/cmd/gateway/filter/sample.yaml b/cmd/gateway/filter/sample.yaml index 790484b297d..1fbe97dbb60 100644 --- a/cmd/gateway/filter/sample.yaml +++ b/cmd/gateway/filter/sample.yaml @@ -98,7 +98,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/cmd/gateway/lb/sample.yaml b/cmd/gateway/lb/sample.yaml index 1fc16c917e2..6b77f3133d8 100644 --- a/cmd/gateway/lb/sample.yaml +++ b/cmd/gateway/lb/sample.yaml @@ -98,7 +98,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/cmd/manager/index/sample.yaml b/cmd/manager/index/sample.yaml index 47b3e1dbde4..d7ef1112fc9 100644 --- a/cmd/manager/index/sample.yaml +++ b/cmd/manager/index/sample.yaml @@ -98,7 +98,7 @@ observability: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false sampling_rate: 1 diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile new file mode 100644 index 00000000000..53c25c6649b --- /dev/null +++ b/dockers/agent/core/faiss/Dockerfile @@ -0,0 +1,120 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG GO_VERSION=latest +ARG MAINTAINER="vdaas.org vald team " + +FROM golang:${GO_VERSION} AS golang + +FROM ubuntu:devel AS builder + +ENV GO111MODULE on +ENV DEBIAN_FRONTEND noninteractive +ENV INITRD No +ENV LANG en_US.UTF-8 +ENV GOROOT /opt/go +ENV GOPATH /go +ENV PATH ${PATH}:${GOROOT}/bin:${GOPATH}/bin +ENV ORG vdaas +ENV REPO vald +ENV PKG agent/core/faiss +ENV PKG_INTERNAL agent/internal +ENV APP_NAME faiss + +# skipcq: DOK-DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + gcc \ + git \ + g++ \ + intel-mkl \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=golang /usr/local/go $GOROOT +RUN mkdir $GOPATH + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} + +COPY go.mod . +COPY go.sum . + +RUN go mod download + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal +COPY internal . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc +COPY apis/grpc . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} +COPY pkg/${PKG} . +# copy ngt/service/kvs and ngt/service/vqueue +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/agent/core/ngt/service +COPY pkg/agent/core/ngt/service/kvs ./kvs +COPY pkg/agent/core/ngt/service/vqueue ./vqueue + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG_INTERNAL} +COPY pkg/${PKG_INTERNAL} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +COPY cmd/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY Makefile . +RUN update-alternatives --set libblas.so-x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/libmkl_rt.so \ + && make faiss/install + +COPY .git . + +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +RUN cp sample.yaml /tmp/config.yaml + +FROM ubuntu:devel +LABEL maintainer "${MAINTAINER}" + +ENV APP_NAME faiss + +COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +COPY --from=builder /tmp/config.yaml /etc/server/config.yaml + +COPY --from=builder /usr/local/lib/libfaiss.so /usr/local/lib/libfaiss.so +COPY --from=builder /lib/x86_64-linux-gnu/libstdc++.so.6 /lib/x86_64-linux-gnu/libstdc++.so.6 +COPY --from=builder /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/x86_64-linux-gnu/libgcc_s.so.1 +COPY --from=builder /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/libc.so.6 +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_intel_lp64.so /lib/x86_64-linux-gnu/libmkl_intel_lp64.so +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_sequential.so /lib/x86_64-linux-gnu/libmkl_sequential.so +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_core.so /lib/x86_64-linux-gnu/libmkl_core.so +COPY --from=builder /lib/x86_64-linux-gnu/libgomp.so.1 /lib/x86_64-linux-gnu/libgomp.so.1 +COPY --from=builder /lib/x86_64-linux-gnu/libm.so.6 /lib/x86_64-linux-gnu/libm.so.6 +COPY --from=builder /lib/x86_64-linux-gnu/libdl.so.2 /lib/x86_64-linux-gnu/libdl.so.2 +COPY --from=builder /lib/x86_64-linux-gnu/libpthread.so.0 /lib/x86_64-linux-gnu/libpthread.so.0 +COPY --from=builder /lib/x86_64-linux-gnu/libmkl_avx2.so /lib/x86_64-linux-gnu/libmkl_avx2.so +RUN ldconfig -v + +ENTRYPOINT ["/go/bin/faiss"] diff --git a/docs/overview/component/README.md b/docs/overview/component/README.md index 49461c6b9bb..58e03e547e5 100644 --- a/docs/overview/component/README.md +++ b/docs/overview/component/README.md @@ -67,6 +67,7 @@ In this section, we will describe what is Vald Agent and the corresponding compo Vald Agent provides functionalities to perform approximate nearest neighbor search. Agent-NGT uses [yahoojapan/NGT](https://github.com/yahoojapan/NGT) as a core library. +And Agent-Faiss uses [facebookresearch/faiss](https://github.com/facebookresearch/faiss) as a core library. Each Vald Agent pod has its own vector data space because only several Vald Agents are selected to be inserted/updated in a single insert/update request. diff --git a/docs/overview/component/agent.md b/docs/overview/component/agent.md index be0326bb7f7..1cf33ad5348 100644 --- a/docs/overview/component/agent.md +++ b/docs/overview/component/agent.md @@ -78,6 +78,56 @@ This image shows the mechanism to create NGT index. Please refer to [Go Doc](https://pkg.go.dev/github.com/vdaas/vald@v1.3.1/pkg/agent/core/ngt/service) for other functions. +#### Vald Agent Faiss + +Vald Agent Faiss uses [Faiss](https://github.com/facebookresearch/faiss) as an algorithm. + +The main functions are the followings: + +- Insert + - Request to insert new vectors into the Faiss. + - Requested vectors are stored in the `vqueue`. + - Cache a fixed number of verctors for Faiss training. + - Once Faiss trained in CreateIndex, the vector is never cached for Faiss training. +- Search + - Get the nearest neighbor vectors of the request vector from Faiss indexes. + - radius/epsilon is search config for NGT and has no meaning in Faiss. +- Update + - Create a request to update the specific vectors to the new vectors. + - Requested vectors are stored in the `vqueue`. +- Remove + - Create a request to remove the specific vectors from Faiss indexes. + - Requested vectors are stored in the `vqueue`. +- Exist + - Check whether the specific vectors are already inserted or not. +- CreateIndex + - Create a new Faiss index structure in memory using vectors stored in the `vqueue` and the existing Faiss index structure if it exists. + - If a certain number of vectors required for Faiss training are not cached, they will not be trained. + - If Faiss is not trained, no index is generated. +- SaveIndex + - Save metadata about Faiss index information to the internal storage. + +Unimplemented functions are the followings: + +- GetObject +- SearchByID +- StreamXXX +- MultiXXX + +
+Same as Agent NGT, You have to control the duration of CreateIndex and SaveIndex by configuration. + +These methods don’t always run when getting the request. + +
+ +
+As you see, Vald Agent Faiss can only search the nearest neighbors from the Faiss index. + +You have to wait to complete the CreateIndex and SaveIndex functions before searching. + +
+ ### Sidecar `Sidecar` saves the index metadata file to external storage like Amazon S3 or Google Cloud Storage. diff --git a/docs/tutorial/get-started-with-faiss-agent.md b/docs/tutorial/get-started-with-faiss-agent.md new file mode 100644 index 00000000000..87d75621601 --- /dev/null +++ b/docs/tutorial/get-started-with-faiss-agent.md @@ -0,0 +1,393 @@ +# Get Started + +This tutorial is for those who have already completed [Get Started](https://github.com/vdaas/vald/blob/main/docs/tutorial/get-started.md). +Please refer to Prepare the Kubernetes Cluster and others there. + +## Deploy Vald on Kubernetes Cluster + +This chapter shows how to deploy Vald using Helm and run it on your Kubernetes cluster.
+In this tutorial, you will deploy the basic configuration of Vald that is consisted of vald-agent-faiss, vald-lb-gateway, vald-discoverer, and vald-manager-index.
+ +1. Clone the repository + + ```bash + git clone https://github.com/vdaas/vald.git && \ + cd vald + ``` + +1. Confirm which cluster to deploy + + ```bash + kubectl cluster-info + ``` + +1. Edit Configurations + + Set the parameters for connecting to the vald-lb-gateway through Kubernetes ingress from the external network. + Please set these parameters. + + ```bash + vim example/helm/values.yaml + === + ## vald-lb-gateway settings + gateway: + lb: + ... + ingress: + enabled: true + # TODO: Set your ingress host. + host: localhost + # TODO: Set annotations which you have to set for your k8s cluster. + annotations: + ... + ## vald-agent-faiss settings + agent: + algorithm: faiss + image: + repository: vdaas/vald-agent-faiss + tag: latest + faiss: + auto_index_check_duration: 1m + auto_index_duration_limit: 24h + auto_index_length: 10 + auto_save_index_duration: 35m + dimension: 784 + enable_copy_on_write: false + enable_in_memory_mode: true + enable_proactive_gc: true + index_path: "" + initial_delay_max_duration: 3m + load_index_timeout_factor: 1ms + m: 8 # dimension % m == 0, train size >= 2^m(or nlist) * minPointsPerCentroid + max_load_index_timeout: 10m + metric_type: "inner_product" + min_load_index_timeout: 3m + nbits_per_idx: 8 + nlist: 100 + ... + ``` + + Note:
+ If you decided to use port-forward instead of ingress, please set `gateway.lb.ingress.enabled` to `false`. + +1. Deploy Vald using Helm + + Add vald repo into the helm repo. + + ```bash + helm repo add vald https://vald.vdaas.org/charts + ``` + + Deploy vald on your Kubernetes cluster. + + ```bash + helm install vald vald/vald --values example/helm/values.yaml + ``` + +1. Verify + + When finish deploying Vald, you can check the Vald's pods status following command. + + ```bash + kubectl get pods + ``` + +
Example output
+ If the deployment is successful, all Vald components should be running. + + ```bash + NAME READY STATUS RESTARTS AGE + vald-agent-faiss-0 1/1 Running 0 7m12s + vald-agent-faiss-1 1/1 Running 0 7m12s + vald-agent-faiss-2 1/1 Running 0 7m12s + vald-agent-faiss-3 1/1 Running 0 7m12s + vald-agent-faiss-4 1/1 Running 0 7m12s + vald-discoverer-7f9f697dbb-q44qh 1/1 Running 0 7m11s + vald-lb-gateway-6b7b9f6948-4z5md 1/1 Running 0 7m12s + vald-lb-gateway-6b7b9f6948-68g94 1/1 Running 0 6m56s + vald-lb-gateway-6b7b9f6948-cvspq 1/1 Running 0 6m56s + vald-manager-index-74c7b5ddd6-jrnlw 1/1 Running 0 7m12s + ``` + +
+ + ```bash + kubectl get ingress + ``` + +
Example output
+ + ```bash + NAME CLASS HOSTS ADDRESS PORTS AGE + vald-lb-gateway-ingress localhost 192.168.16.2 80 7m43s + ``` + +
+ + ```bash + kubectl get svc + ``` + +
Example output
+ + ```bash + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kubernetes ClusterIP 10.43.0.1 443/TCP 9m29s + vald-agent-faiss ClusterIP None 8081/TCP,3001/TCP 8m48s + vald-discoverer ClusterIP None 8081/TCP,3001/TCP 8m48s + vald-manager-index ClusterIP None 8081/TCP,3001/TCP 8m48s + vald-lb-gateway ClusterIP None 8081/TCP,3001/TCP 8m48s + ``` + +
+ +## Run Example Code + +In this chapter, you will execute insert, search, and delete vectors to your Vald cluster using the example code.
+The [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) is used as a dataset for indexing and search query. + +The example code is implemented in Go and using [vald-client-go](https://github.com/vdaas/vald-client-go), one of the official Vald client libraries, for requesting to Vald cluster. +Vald provides multiple language client libraries such as Go, Java, Node.js, Python, etc. +If you are interested, please refer to [SDKs](../user-guides/sdks.md).
+ +1. Port Forward(option) + + If you do not use Kubernetes Ingress, port-forward is required to make requests from your local environment. + + ```bash + kubectl port-forward deployment/vald-lb-gateway 8081:8081 + ``` + +1. Download dataset + + Download [Fashion-MNIST](https://github.com/zalandoresearch/fashion-mnist) that is used as a dataset for indexing and search query. + + Move to the working directory + + ```bash + cd example/client + ``` + + Download Fashion-MNIST testing dataset + + ```bash + wget http://ann-benchmarks.com/fashion-mnist-784-euclidean.hdf5 + ``` + +1. Run Example + + We use [`example/client/main.go`](https://github.com/vdaas/vald/blob/main/example/client/main.go) to run the example.
+ This example will insert and index 400 vectors into the Vald from the Fashion-MNIST dataset via [gRPC](https://grpc.io/). + And then after waiting for indexing, it will request for searching the nearest vector 10 times. + You will get the 10 nearest neighbor vectors for each search query.
+ Run example codes by executing the below command. + + ```bash + go run main.go + ``` + +
The detailed explanation of example code is here
+ This will execute 6 steps. + + 1. init + + - Import packages +
example code
+ + ```go + package main + + import ( + "context" + "encoding/json" + "flag" + "time" + + "github.com/kpango/fuid" + "github.com/kpango/glg" + "github.com/vdaas/vald-client-go/v1/payload" + "github.com/vdaas/vald-client-go/v1/vald" + + "gonum.org/v1/hdf5" + "google.golang.org/grpc" + ) + ``` + +
+ + - Set variables + + - The constant number of training datasets and test datasets. +
example code
+ + ```go + const ( + insertCount = 400 + testCount = 20 + ) + ``` + +
+ + - The variables for configuration. +
example code
+ + ```go + const ( + datasetPath string + grpcServerAddr string + indexingWaitSeconds uint + ) + ``` + +
+ + - Recognition parameters. +
example code
+ + ```go + func init() { + flag.StringVar(&datasetPath, "path", "fashion-mnist-784-euclidean.hdf5", "set dataset path") + flag.StringVar(&grpcServerAddr, "addr", "127.0.0.1:8081", "set gRPC server address") + flag.UintVar(&indexingWaitSeconds, "wait", 60, "set indexing wait seconds") + flag.Parse() + } + ``` + +
+ + 1. load + + - Loading from Fashion-MNIST dataset and set id for each vector that is loaded. This step will return the training dataset, test dataset, and ids list of ids when loading is completed with success. +
example code
+ + ```go + ids, train, test, err := load(datasetPath) + if err != nil { + glg.Fatal(err) + } + ``` + +
+ + 1. Create the gRPC connection and Vald client with gRPC connection. + +
example code
+ + ```go + ctx := context.Background() + + conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithInsecure()) + if err != nil { + glg.Fatal(err) + } + + client := vald.NewValdClient(conn) + ``` + +
+ + 1. Insert and Index + + - Insert and Indexing 400 training datasets to the Vald agent. +
example code
+ + ```go + for i := range ids [:insertCount] { + _, err := client.Insert(ctx, &payload.Insert_Request{ + Vector: &payload.Object_Vector{ + Id: ids[i], + Vector: train[i], + }, + Config: &payload.Insert_Config{ + SkipStrictExistCheck: true, + }, + }) + if err != nil { + glg.Fatal(err) + } + if i%10 == 0 { + glg.Infof("Inserted %d", i) + } + } + ``` + +
+ + - Wait until indexing finish. +
example code
+ + ```go + wt := time.Duration(indexingWaitSeconds) * time.Second + glg.Infof("Wait %s for indexing to finish", wt) + time.Sleep(wt) + ``` + +
+ + 1. Search + + - Search 10 neighbor vectors for each 20 test datasets and return a list of the neighbor vectors. + + - When getting approximate vectors, the Vald client sends search config and vector to the server via gRPC. +
example code
+ + ```go + glg.Infof("Start search %d times", testCount) + for i, vec := range test[:testCount] { + res, err := client.Search(ctx, &payload.Search_Request){ + Vector: vec, + Config: &payload.Search_Config{ + Num: 10, + Radius: -1, + Epsilon: 0.1, + Timeout: 100000000, + } + } + if err != nil { + glg.Fatal(err) + } + + b, _ := json.MarshalIndent(res.GetResults(), "", " ") + glg.Infof("%d - Results : %s\n\n", i+1, string(b)) + time.Sleep(1 * time.Second) + } + ``` + +
+ + 1. Remove + + - Remove 400 indexed training datasets from the Vald agent. +
example code
+ + ```go + for i := range ids [:insertCount] { + _, err := client.Remove(ctx, &payload.Remove_Request{ + Id: &payload.Object_ID{ + Id: ids[i], + }, + }) + if err != nil { + glg.Fatal(err) + } + if i%10 == 0 { + glg.Infof("Removed %d", i) + } + } + ``` + +
+ +## Cleanup + +In the last, you can remove the deployed Vald Cluster by executing the below command. + +```bash +helm uninstall vald +``` + +## References + +- [Get Started with NGT agent by default](https://github.com/vdaas/vald/blob/main/docs/tutorial/get-started.md) +- [Faiss](https://github.com/facebookresearch/faiss) diff --git a/docs/user-guides/backup-configuration.md b/docs/user-guides/backup-configuration.md index 0f8872f38f8..06c1e3b21ab 100644 --- a/docs/user-guides/backup-configuration.md +++ b/docs/user-guides/backup-configuration.md @@ -29,7 +29,7 @@ Please refer to the following tables and decide which method fit for your case. This section shows the best practice for configuring backup features with PV, S3, or PV + S3. -Each sample configuration yaml is published on [here](https://github.com/vdaas/vald/tree/master/charts/vald/values). +Each sample configuration yaml is published on [here](https://github.com/vdaas/vald/tree/main/charts/vald/values). Please refer it for more details. ### General diff --git a/docs/user-guides/configuration.md b/docs/user-guides/configuration.md index b3b9e06d882..f54def2a171 100644 --- a/docs/user-guides/configuration.md +++ b/docs/user-guides/configuration.md @@ -175,6 +175,42 @@ When the setting parameter of Vald Agent NGT is shorter than the setting value o If this happens, the Index Manager may not function properly. +#### Faiss + +Vald Agent Faiss uses [facebookresearch/faiss][faiss] as a core library for searching vectors. +The behaviors of Faiss can be configured by setting `agent.faiss` field object. + +The important parameters are the followings: + +- `agent.faiss.dimension` +- `agent.faiss.distance_type` +- `agent.faiss.m` +- `agent.faiss.metric_type` +- `agent.faiss.nbits_per_idx` +- `agent.faiss.nlist` + +Users should configure these parameters first to fit their use case. +For further details, please read [the Fiass wiki][faiss-wiki]. + +Vald Agent Faiss has a feature to start indexing automatically. +The behavior of this feature can be configured with these parameters: + +- `agent.faiss.auto_index_duration_limit` +- `agent.faiss.auto_index_check_duration` +- `agent.faiss.auto_index_length` + +
+While the Vald Agent Faiss is in the process of creating indexes, it will ignore all search requests to the target pods. +
+ +
+When deploying Vald Index Manager, the above parameters should be set much longer than the Vald Index Manager settings (Please refer to the Vald Index Manager section) or minus value.
+E.g., set agent.faiss.auto_index_duration_limit to "720h" or "-1h" and agent.faiss.auto_index_check_duration to "24h" or "-1h".
+This is because the Vald Index Manager accurately grasps the index information of each Vald Agent Faiss and controls the execution timing of indexing.

+When the setting parameter of Vald Agent Faiss is shorter than the setting value of Vald Index Manager, Vald Agent Faiss may start indexing by itself without the execution command from Vald Index Manager. +If this happens, the Index Manager may not function properly. +
+ #### Resource requests and limits, Pod priorities Because the Vald Agent pod places indexes on memory, termination of agent pods causes loss of indexes. @@ -371,3 +407,5 @@ For further details, there are references to the Helm values in the Vald GitHub [kubernetes-topology-spread-constraints]: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ [yj-ngt]: https://github.com/yahoojapan/NGT [yj-ngt-wiki]: https://github.com/yahoojapan/NGT/wiki +[faiss]: https://github.com/facebookresearch/faiss +[faiss-wiki]: https://github.com/facebookresearch/faiss/wiki diff --git a/docs/user-guides/observability-configuration.md b/docs/user-guides/observability-configuration.md index cf2734ab9e2..66617e874b2 100644 --- a/docs/user-guides/observability-configuration.md +++ b/docs/user-guides/observability-configuration.md @@ -132,7 +132,7 @@ defaults: - "go_version" - "go_os" - "go_arch" - - "ngt_version" + - "algorithm_info" # enable memory metrics enable_memory: true # enable goroutine metrics diff --git a/example/client/go.mod b/example/client/go.mod index a8e2c37c62b..d1f32f316b9 100644 --- a/example/client/go.mod +++ b/example/client/go.mod @@ -11,9 +11,9 @@ replace ( golang.org/x/crypto => golang.org/x/crypto v0.10.0 golang.org/x/net => golang.org/x/net v0.11.0 golang.org/x/text => golang.org/x/text v0.10.0 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230626202813-9b080da550b3 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 google.golang.org/grpc => google.golang.org/grpc v1.56.1 google.golang.org/protobuf => google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 diff --git a/example/client/go.sum b/example/client/go.sum index 9b9963489ac..50d1a8dfd38 100644 --- a/example/client/go.sum +++ b/example/client/go.sum @@ -27,12 +27,12 @@ golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= -google.golang.org/genproto v0.0.0-20230626202813-9b080da550b3 h1:Yofj1/U0xc/Zi5KEpoIxm51I2f85X+eGyY4YzAujRdw= -google.golang.org/genproto v0.0.0-20230626202813-9b080da550b3/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3 h1:wl7z+A0jkB3Rl8Hz74SqGDlnnn5VlL2CV+9UTdZOo00= -google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3 h1:QJuqz7YzNTyKDspkp2lrzqtq4lf2AhUSpXTsGP5SbLw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk= +google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= diff --git a/example/client/main.go b/example/client/main.go index 3a608b41d57..335c693eb66 100644 --- a/example/client/main.go +++ b/example/client/main.go @@ -28,24 +28,26 @@ import ( ) const ( - insertCount = 400 - testCount = 20 + testCount = 20 ) var ( datasetPath string grpcServerAddr string + insertCount uint indexingWaitSeconds uint ) func init() { /** Path option specifies hdf file by path. Default value is `fashion-mnist-784-euclidean.hdf5`. - Addr option specifies grpc server address. Default value is `127.0.0.1:8080`. - Wait option specifies indexing wait time (in seconds). Default value is `60`. + Addr option specifies grpc server address. Default value is `127.0.0.1:8081`. + Insert option specifies insert count. Default value is `400`. + Wait option specifies indexing wait time (in seconds). Default value is `60`. **/ flag.StringVar(&datasetPath, "path", "fashion-mnist-784-euclidean.hdf5", "dataset path") - flag.StringVar(&grpcServerAddr, "addr", "localhost:8080", "gRPC server address") + flag.StringVar(&grpcServerAddr, "addr", "localhost:8081", "gRPC server address") + flag.UintVar(&insertCount, "insert", 400, "insert count") flag.UintVar(&indexingWaitSeconds, "wait", 60, "indexing wait seconds") flag.Parse() } diff --git a/go.mod b/go.mod index aeccbce84f9..95b4cdb5680 100755 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ replace ( github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b github.com/antihax/optional => github.com/antihax/optional v1.0.0 github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.293 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.295 github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.18.1 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.27 @@ -190,7 +190,7 @@ replace ( github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v1.0.0 github.com/kisielk/errcheck => github.com/kisielk/errcheck v1.6.3 github.com/kisielk/gotool => github.com/kisielk/gotool v1.0.0 - github.com/klauspost/compress => github.com/klauspost/compress v1.16.7-0.20230622111944-67a538e2b4df + github.com/klauspost/compress => github.com/klauspost/compress v1.16.7 github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.5 github.com/kpango/fastime => github.com/kpango/fastime v1.1.9 github.com/kpango/fuid => github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 @@ -244,7 +244,7 @@ replace ( github.com/quasilyte/gogrep => github.com/quasilyte/gogrep v0.5.0 github.com/quasilyte/stdinfo => github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid v1.2.0 - github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal v1.10.0 + github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal v1.11.0 github.com/rs/xid => github.com/rs/xid v1.5.0 github.com/rs/zerolog => github.com/rs/zerolog v1.29.1 github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 v2.1.0 @@ -271,17 +271,17 @@ replace ( github.com/zeebo/assert => github.com/zeebo/assert v1.3.1 github.com/zeebo/xxh3 => github.com/zeebo/xxh3 v1.0.2 go.opencensus.io => go.opencensus.io v0.24.0 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 - go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric => go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 - go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.33.0 - go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.11.1 - go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.33.0 - go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 + go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric => go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 + go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v1.16.0 + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.39.0 + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.16.0 go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.20.0 go.starlark.net => go.starlark.net v0.0.0-20230612165344-9532f5667272 go.uber.org/atomic => go.uber.org/atomic v1.11.0 @@ -372,14 +372,14 @@ require ( github.com/scylladb/gocqlx v0.0.0-00010101000000-000000000000 github.com/zeebo/xxh3 v1.0.2 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 - go.opentelemetry.io/otel v1.11.2 + go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.0.0-00010101000000-000000000000 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 - go.opentelemetry.io/otel/metric v0.34.0 - go.opentelemetry.io/otel/sdk v1.11.1 - go.opentelemetry.io/otel/sdk/metric v0.33.0 - go.opentelemetry.io/otel/trace v1.11.2 + go.opentelemetry.io/otel/metric v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/sdk/metric v0.39.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/automaxprocs v0.0.0-00010101000000-000000000000 go.uber.org/goleak v1.2.1 go.uber.org/zap v1.24.0 @@ -417,7 +417,7 @@ require ( github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/campoy/embedmd v1.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -480,8 +480,8 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.11.0 // indirect diff --git a/go.sum b/go.sum index 5c96c6a078b..5d763c76b3b 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/aws/aws-sdk-go v1.44.293 h1:oBPrQqsyMYe61Sl/xKVvQFflXjPwYH11aKi8QR3Nhts= -github.com/aws/aws-sdk-go v1.44.293/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.295 h1:SGjU1+MqttXfRiWHD6WU0DRhaanJgAFY+xIhEaugV8Y= +github.com/aws/aws-sdk-go v1.44.295/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= @@ -260,6 +260,7 @@ github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY= github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -287,6 +288,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= @@ -495,8 +497,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.6.3/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.16.7-0.20230622111944-67a538e2b4df h1:YVmtlF3q1+H7fzHO2iCZ6n/LmfF3HfqshO/jIqbqpRU= -github.com/klauspost/compress v1.16.7-0.20230622111944-67a538e2b4df/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84= @@ -604,8 +606,8 @@ github.com/rakyll/embedmd v0.0.0-20171029212350-c8060a0752a2/go.mod h1:7jOTMgqac github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -654,28 +656,28 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= -go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4= -go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 h1:X2GndnMCsUPh6CiY2a+frAbNsXaPLbB0soHRYhAZ5Ig= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1/go.mod h1:i8vjiSzbiUC7wOQplijSXMYUpNM93DtlS5CbUT+C6oQ= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.33.0 h1:OT/UjHcjog4A1s1UMCtyehIKS+vpjM5Du0r7KGsH6TE= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.33.0/go.mod h1:0XctNDHEWmiSDIU8NPbJElrK05gBJFcYlGP4FMGo4g4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.33.0 h1:1SVtGtRsNyGgv1fRfNXfh+sJowIwzF0gkf+61lvTgdg= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.33.0/go.mod h1:ryB27ubOBXsiqfh6MwtSdx5knzbSZtjvPnMMmt3AykQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 h1:MEQNafcNCB0uQIti/oHgU7CZpUMYQ7qigBwMVKycHvc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1/go.mod h1:19O5I2U5iys38SsmT2uDJja/300woyzE1KPIQxEUBUc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 h1:LYyG/f1W/jzAix16jbksJfMQFpOH/Ma6T639pVPMgfI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1/go.mod h1:QrRRQiY3kzAoYPNLP0W/Ikg0gR6V3LMc+ODSxr7yyvg= -go.opentelemetry.io/otel/metric v0.33.0 h1:xQAyl7uGEYvrLAiV/09iTJlp1pZnQ9Wl793qbVvED1E= -go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI= -go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs= -go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys= -go.opentelemetry.io/otel/sdk/metric v0.33.0 h1:oTqyWfksgKoJmbrs2q7O7ahkJzt+Ipekihf8vhpa9qo= -go.opentelemetry.io/otel/sdk/metric v0.33.0/go.mod h1:xdypMeA21JBOvjjzDUtD0kzIcHO/SPez+a8HOzJPGp0= -go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= -go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 h1:f6BwB2OACc3FCbYVznctQ9V6KK7Vq6CjmYXJ7DeSs4E= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0/go.mod h1:UqL5mZ3qs6XYhDnZaW1Ps4upD+PX6LipH40AoeuIlwU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 h1:rm+Fizi7lTM2UefJ1TO347fSRcwmIsUAaZmYmIGBRAo= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0/go.mod h1:sWFbI3jJ+6JdjOVepA5blpv/TJ20Hw+26561iMbWcwU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI= +go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.20.0 h1:BLOA1cZBAGSbRiNuGCCKiFrCdYB7deeHDeD1SueyOfA= go.opentelemetry.io/proto/otlp v0.20.0/go.mod h1:3QgjzPALBIv9pcknj2EXGPXjYPFdUh/RQfF8Lz3+Vnw= go.starlark.net v0.0.0-20230612165344-9532f5667272 h1:2/wtqS591wZyD2OsClsVBKRPEvBsQt/Js+fsCiYhwu8= diff --git a/hack/go.mod.default b/hack/go.mod.default index f91d21d04ba..6a72a509a5c 100755 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -271,17 +271,17 @@ replace ( github.com/zeebo/assert => github.com/zeebo/assert latest github.com/zeebo/xxh3 => github.com/zeebo/xxh3 latest go.opencensus.io => go.opencensus.io latest - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 - go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric => go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 - go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.33.0 - go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.11.1 - go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.33.0 - go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc latest + go.opentelemetry.io/otel => go.opentelemetry.io/otel latest + go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry latest + go.opentelemetry.io/otel/exporters/otlp/otlpmetric => go.opentelemetry.io/otel/exporters/otlp/otlpmetric latest + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc => go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc latest + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace latest + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc latest + go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric latest + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk latest + go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric latest + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace latest go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp latest go.starlark.net => go.starlark.net latest go.uber.org/atomic => go.uber.org/atomic latest diff --git a/hack/license/gen/main.go b/hack/license/gen/main.go index 6a3d5ce9000..b502bfd40a7 100644 --- a/hack/license/gen/main.go +++ b/hack/license/gen/main.go @@ -141,6 +141,7 @@ func dirwalk(dir string) []string { "CONTRIBUTORS", "GO_VERSION", "NGT_VERSION", + "FAISS_VERSION", "Pipefile", "VALD_VERSION", "grp", @@ -190,7 +191,7 @@ func readAndRewrite(path string) error { } } else { switch filepath.Ext(path) { - case ".go", ".proto": + case ".go", ".proto", ".c", ".h", ".hpp", ".cpp": d.Escape = slushEscape } lf := true diff --git a/internal/client/v1/client/agent/core/client.go b/internal/client/v1/client/agent/core/client.go index 2b70cbafc43..bb2dde283cf 100644 --- a/internal/client/v1/client/agent/core/client.go +++ b/internal/client/v1/client/agent/core/client.go @@ -86,11 +86,7 @@ func New(opts ...Option) (Client, error) { return c, nil } -func NewAgentClient(cc *grpc.ClientConn) interface { - vald.Client - client.ObjectReader - client.Indexer -} { +func NewAgentClient(cc *grpc.ClientConn) Client { return &singleAgentClient{ Client: vald.NewValdClient(cc), ac: agent.NewAgentClient(cc), diff --git a/internal/config/faiss.go b/internal/config/faiss.go new file mode 100644 index 00000000000..732d5a26be1 --- /dev/null +++ b/internal/config/faiss.go @@ -0,0 +1,118 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package config providers configuration type and load configuration logic +package config + +// Faiss represent the faiss core configuration for server. +type Faiss struct { + // IndexPath represents the faiss index file path + IndexPath string `yaml:"index_path" json:"index_path,omitempty"` + + // Dimension represents the faiss index dimension + Dimension int `yaml:"dimension" json:"dimension,omitempty" info:"dimension"` + + // Nlist represents the number of Voronoi cells + // ref: https://github.com/facebookresearch/faiss/wiki/Faster-search + Nlist int `yaml:"nlist" json:"nlist,omitempty" info:"nlist"` + + // M represents the number of subquantizers + // ref: https://github.com/facebookresearch/faiss/wiki/Faiss-indexes-(composite)#cell-probe-method-with-a-pq-index-as-coarse-quantizer + M int `yaml:"m" json:"m,omitempty" info:"m"` + + // NbitsPerIdx represents the number of bit per subvector index + // ref: https://github.com/facebookresearch/faiss/wiki/FAQ#can-i-ignore-warning-clustering-xxx-points-to-yyy-centroids + NbitsPerIdx int `yaml:"nbits_per_idx" json:"nbits_per_idx,omitempty" info:"nbits_per_idx"` + + // MetricType represents the metric type + MetricType string `yaml:"metric_type" json:"metric_type,omitempty" info:"metric_type"` + + // EnableInMemoryMode enables on memory faiss indexing mode + EnableInMemoryMode bool `yaml:"enable_in_memory_mode" json:"enable_in_memory_mode,omitempty"` + + // AutoIndexCheckDuration represents checking loop duration about auto indexing execution + AutoIndexCheckDuration string `yaml:"auto_index_check_duration" json:"auto_index_check_duration,omitempty"` + + // AutoSaveIndexDuration represents checking loop duration about auto save index execution + AutoSaveIndexDuration string `yaml:"auto_save_index_duration" json:"auto_save_index_duration,omitempty"` + + // AutoIndexDurationLimit represents auto indexing duration limit + AutoIndexDurationLimit string `yaml:"auto_index_duration_limit" json:"auto_index_duration_limit,omitempty"` + + // AutoIndexLength represents auto index length limit + AutoIndexLength int `yaml:"auto_index_length" json:"auto_index_length,omitempty"` + + // InitialDelayMaxDuration represents maximum duration for initial delay + InitialDelayMaxDuration string `yaml:"initial_delay_max_duration" json:"initial_delay_max_duration,omitempty"` + + // MinLoadIndexTimeout represents minimum duration of load index timeout + MinLoadIndexTimeout string `yaml:"min_load_index_timeout" json:"min_load_index_timeout,omitempty"` + + // MaxLoadIndexTimeout represents maximum duration of load index timeout + MaxLoadIndexTimeout string `yaml:"max_load_index_timeout" json:"max_load_index_timeout,omitempty"` + + // LoadIndexTimeoutFactor represents a factor of load index timeout + LoadIndexTimeoutFactor string `yaml:"load_index_timeout_factor" json:"load_index_timeout_factor,omitempty"` + + // EnableProactiveGC enables more proactive GC call for reducing heap memory allocation + EnableProactiveGC bool `yaml:"enable_proactive_gc" json:"enable_proactive_gc,omitempty"` + + // EnableCopyOnWrite enables copy on write saving + EnableCopyOnWrite bool `yaml:"enable_copy_on_write" json:"enable_copy_on_write,omitempty"` + + // VQueue represents the faiss vector queue buffer size + VQueue *VQueue `json:"vqueue,omitempty" yaml:"vqueue"` + + // KVSDB represents the faiss bidirectional kv store configuration + KVSDB *KVSDB `json:"kvsdb,omitempty" yaml:"kvsdb"` +} + +//// KVSDB represent the faiss vector bidirectional kv store configuration +//type KVSDB struct { +// // Concurrency represents kvsdb range loop processing concurrency +// Concurrency int `json:"concurrency,omitempty" yaml:"concurrency,omitempty"` +//} +// +//// VQueue represent the faiss vector queue buffer size +//type VQueue struct { +// // InsertBufferPoolSize represents insert time ordered slice buffer size +// InsertBufferPoolSize int `json:"insert_buffer_pool_size,omitempty" yaml:"insert_buffer_pool_size"` +// +// // DeleteBufferPoolSize represents delete time ordered slice buffer size +// DeleteBufferPoolSize int `json:"delete_buffer_pool_size,omitempty" yaml:"delete_buffer_pool_size"` +//} + +// Bind returns Faiss object whose some string value is filed value or environment value. +func (f *Faiss) Bind() *Faiss { + f.IndexPath = GetActualValue(f.IndexPath) + f.MetricType = GetActualValue(f.MetricType) + f.AutoIndexCheckDuration = GetActualValue(f.AutoIndexCheckDuration) + f.AutoIndexDurationLimit = GetActualValue(f.AutoIndexDurationLimit) + f.AutoSaveIndexDuration = GetActualValue(f.AutoSaveIndexDuration) + f.InitialDelayMaxDuration = GetActualValue(f.InitialDelayMaxDuration) + f.MinLoadIndexTimeout = GetActualValue(f.MinLoadIndexTimeout) + f.MaxLoadIndexTimeout = GetActualValue(f.MaxLoadIndexTimeout) + f.LoadIndexTimeoutFactor = GetActualValue(f.LoadIndexTimeoutFactor) + + if f.VQueue == nil { + f.VQueue = new(VQueue) + } + if f.KVSDB == nil { + f.KVSDB = new(KVSDB) + } + + return f +} diff --git a/internal/config/server.go b/internal/config/server.go index c8ba7274465..4285a01e592 100644 --- a/internal/config/server.go +++ b/internal/config/server.go @@ -20,6 +20,7 @@ package config import ( "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/admin" "github.com/vdaas/vald/internal/net/grpc/health" "github.com/vdaas/vald/internal/net/grpc/reflection" "github.com/vdaas/vald/internal/servers/server" @@ -89,6 +90,7 @@ type GRPC struct { HeaderTableSize int `json:"header_table_size,omitempty" yaml:"header_table_size"` Interceptors []string `json:"interceptors,omitempty" yaml:"interceptors"` EnableReflection bool `json:"enable_reflection,omitempty" yaml:"enable_reflection"` + EnableAdmin bool `json:"enable_admin,omitempty" yaml:"enable_admin"` } // GRPCKeepalive represents the configuration for gRPC keep-alive. @@ -282,6 +284,12 @@ func (s *Server) Opts() []server.Option { reflection.Register(srv) })) } + if s.GRPC.EnableAdmin { + opts = append(opts, + server.WithGRPCRegistFunc(func(srv *grpc.Server) { + admin.Register(srv) + })) + } if s.GRPC.Keepalive != nil { opts = append(opts, server.WithGRPCKeepaliveMaxConnIdle(s.GRPC.Keepalive.MaxConnIdle), diff --git a/internal/core/algorithm/faiss/Capi.cpp b/internal/core/algorithm/faiss/Capi.cpp new file mode 100644 index 00000000000..85cdbc7ad3d --- /dev/null +++ b/internal/core/algorithm/faiss/Capi.cpp @@ -0,0 +1,231 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Capi.h" + +FaissStruct* faiss_create_index( + const int d, + const int nlist, + const int m, + const int nbits_per_idx, + const int metric_type) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + FaissStruct *st = NULL; + try { + faiss::IndexFlat *quantizer; + switch (metric_type) { + case faiss::METRIC_INNER_PRODUCT: + quantizer = new faiss::IndexFlat(d, faiss::METRIC_INNER_PRODUCT); + break; + case faiss::METRIC_L2: + quantizer = new faiss::IndexFlat(d, faiss::METRIC_L2); + break; + default: + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: no metric type."; + std::cerr << ss.str() << std::endl; + return NULL; + } + faiss::IndexIVFPQ *index = new faiss::IndexIVFPQ(quantizer, d, nlist, m, nbits_per_idx); + //index->verbose = true; + st = new FaissStruct{ + static_cast(quantizer), + static_cast(index) + }; + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + } + + return st; +} + +FaissStruct* faiss_read_index(const char* fname) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + FaissStruct *st = NULL; + try { + st = new FaissStruct{ + static_cast(NULL), + static_cast(faiss::read_index(fname)) + }; + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + } + + return st; +} + +bool faiss_write_index( + const FaissStruct* st, + const char* fname) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + try { + faiss::write_index(static_cast(st->faiss_index), fname); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return false; + } + + fflush(stdout); + return true; +} + +bool faiss_train( + const FaissStruct* st, + const int nb, + const float* xb) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + try { + //printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + //printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + (static_cast(st->faiss_index))->train(nb, xb); + //printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + //printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return false; + } + + fflush(stdout); + return true; +} + +int faiss_add( + const FaissStruct* st, + const int nb, + const float* xb, + const long int* xids ) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + try { + //printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + //printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + (static_cast(st->faiss_index))->add_with_ids(nb, xb, xids); + //printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + //printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return -1; + } + + fflush(stdout); + return (static_cast(st->faiss_index))->ntotal; +} + +bool faiss_search( + const FaissStruct* st, + const int k, + const int nq, + const float* xq, + long* I, + float* D) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + try { + //printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + //printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + (static_cast(st->faiss_index))->search(nq, xq, k, D, I); + //printf("I=\n"); + //for(int i = 0; i < nq; i++) { + // for(int j = 0; j < k; j++) { + // printf("%5ld ", I[i * k + j]); + // } + // printf("\n"); + //} + //printf("D=\n"); + //for(int i = 0; i < nq; i++) { + // for(int j = 0; j < k; j++) { + // printf("%7g ", D[i * k + j]); + // } + // printf("\n"); + //} + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return false; + } + + return true; +} + +int faiss_remove( + const FaissStruct* st, + const int size, + const long int* ids) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + try { + //printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + //printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + faiss::IDSelectorArray sel(size, ids); + (static_cast(st->faiss_index))->remove_ids(sel); + //printf("is_trained: %d\n", (static_cast(st->faiss_index))->is_trained); + //printf("ntotal: %ld\n", (static_cast(st->faiss_index))->ntotal); + } catch(std::exception &err) { + std::stringstream ss; + ss << "Capi : " << __FUNCTION__ << "() : Error: " << err.what(); + std::cerr << ss.str() << std::endl; + return -1; + } + + return (static_cast(st->faiss_index))->ntotal; +} + +void faiss_free(FaissStruct* st) { + //printf(__FUNCTION__); + //printf("\n"); + //fflush(stdout); + + free(st); + return; +} diff --git a/internal/core/algorithm/faiss/Capi.h b/internal/core/algorithm/faiss/Capi.h new file mode 100644 index 00000000000..c1df817d359 --- /dev/null +++ b/internal/core/algorithm/faiss/Capi.h @@ -0,0 +1,64 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifdef __cplusplus +extern "C" { +#endif + #include + #include + #include + + typedef void* FaissQuantizer; + typedef void* FaissIndex; + typedef struct { + FaissQuantizer faiss_quantizer; + FaissIndex faiss_index; + } FaissStruct; + + FaissStruct* faiss_create_index( + const int d, + const int nlist, + const int m, + const int nbits_per_idx, + const int metric_type); + FaissStruct* faiss_read_index(const char* fname); + bool faiss_write_index( + const FaissStruct* st, + const char* fname); + bool faiss_train( + const FaissStruct* st, + const int nb, + const float* xb); + int faiss_add( + const FaissStruct* st, + const int nb, + const float* xb, + const long int* xids); + bool faiss_search( + const FaissStruct* st, + const int k, + const int nq, + const float* xq, + long* I, + float* D); + int faiss_remove( + const FaissStruct* st, + const int size, + const long int* ids); + void faiss_free(FaissStruct* st); +#ifdef __cplusplus +} +#endif diff --git a/internal/core/algorithm/faiss/faiss.go b/internal/core/algorithm/faiss/faiss.go new file mode 100644 index 00000000000..b7a0e605186 --- /dev/null +++ b/internal/core/algorithm/faiss/faiss.go @@ -0,0 +1,255 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package faiss provides implementation of Go API for https://github.com/facebookresearch/faiss +package faiss + +/* +#cgo LDFLAGS: -lfaiss +#include +*/ +import "C" + +import ( + "sync" + "unsafe" + + "github.com/vdaas/vald/internal/errors" +) + +type ( + // Faiss is core interface. + Faiss interface { + // SaveIndex stores faiss index to strage. + SaveIndex() error + + // SaveIndexWithPath stores faiss index to specified storage. + SaveIndexWithPath(idxPath string) error + + // Train trains faiss index. + Train(nb int, xb []float32) error + + // Add returns faiss ntotal. + Add(nb int, xb []float32, xids []int64) (int, error) + + // Search returns search result as []SearchResult. + Search(k, nq int, xq []float32) ([]SearchResult, error) + + // Remove removes from faiss index. + Remove(size int, ids []int64) (int, error) + + // Close faiss index. + Close() + } + + faiss struct { + st *C.FaissStruct + dimension C.int + nlist C.int + m C.int + nbitsPerIdx C.int + metricType metricType + idxPath string + mu *sync.RWMutex + } + + SearchResult struct { + ID uint32 + Distance float32 + Error error + } +) + +// metricType is alias of metric type in Faiss. +type metricType int + +const ( + // ------------------------------------------------------------- + // Metric Type Definition + // (https://github.com/facebookresearch/faiss/wiki/MetricType-and-distances) + // ------------------------------------------------------------- + // DistanceNone is unknown distance type. + DistanceNone metricType = iota - 1 + // InnerProduct is inner product. + InnerProduct + // L2 is l2 norm. + L2 + // -------------------------------------------------------------. + + // ------------------------------------------------------------- + // ErrorCode is false + // -------------------------------------------------------------. + ErrorCode = C._Bool(false) + // -------------------------------------------------------------. +) + +// New returns Faiss instance with recreating empty index file. +func New(opts ...Option) (Faiss, error) { + return gen(false, opts...) +} + +func Load(opts ...Option) (Faiss, error) { + return gen(true, opts...) +} + +func gen(isLoad bool, opts ...Option) (Faiss, error) { + var ( + f = new(faiss) + err error + ) + f.mu = new(sync.RWMutex) + + defer func() { + if err != nil { + f.Close() + } + }() + + for _, opt := range append(defaultOptions, opts...) { + if err = opt(f); err != nil { + return nil, errors.NewFaissError("faiss option error") + } + } + + if isLoad { + path := C.CString(f.idxPath) + defer C.free(unsafe.Pointer(path)) + f.st = C.faiss_read_index(path) + if f.st == nil { + return nil, errors.NewFaissError("faiss load index error") + } + } else { + switch f.metricType { + case InnerProduct: + f.st = C.faiss_create_index(f.dimension, f.nlist, f.m, f.nbitsPerIdx, C.int(InnerProduct)) + case L2: + f.st = C.faiss_create_index(f.dimension, f.nlist, f.m, f.nbitsPerIdx, C.int(L2)) + default: + return nil, errors.NewFaissError("faiss create index error: no metric type") + } + if f.st == nil { + return nil, errors.NewFaissError("faiss create index error: nil pointer") + } + } + + return f, nil +} + +// SaveIndex stores faiss index to storage. +func (f *faiss) SaveIndex() error { + path := C.CString(f.idxPath) + defer C.free(unsafe.Pointer(path)) + + f.mu.Lock() + ret := C.faiss_write_index(f.st, path) + f.mu.Unlock() + if ret == ErrorCode { + return errors.NewFaissError("failed to faiss_write_index") + } + + return nil +} + +// SaveIndexWithPath stores faiss index to specified storage. +func (f *faiss) SaveIndexWithPath(idxPath string) error { + path := C.CString(idxPath) + defer C.free(unsafe.Pointer(path)) + + f.mu.Lock() + ret := C.faiss_write_index(f.st, path) + f.mu.Unlock() + if ret == ErrorCode { + return errors.NewFaissError("failed to faiss_write_index") + } + + return nil +} + +// Train trains faiss index. +func (f *faiss) Train(nb int, xb []float32) error { + f.mu.Lock() + ret := C.faiss_train(f.st, (C.int)(nb), (*C.float)(&xb[0])) + f.mu.Unlock() + if ret == ErrorCode { + return errors.NewFaissError("failed to faiss_train") + } + + return nil +} + +// Add returns faiss ntotal. +func (f *faiss) Add(nb int, xb []float32, xids []int64) (int, error) { + dim := int(f.dimension) + if len(xb) != dim*nb || len(xb) != dim*len(xids) { + return -1, errors.ErrIncompatibleDimensionSize(len(xb)/nb, dim) + } + + f.mu.Lock() + ntotal := int(C.faiss_add(f.st, (C.int)(nb), (*C.float)(&xb[0]), (*C.long)(&xids[0]))) + f.mu.Unlock() + if ntotal < 0 { + return ntotal, errors.NewFaissError("failed to faiss_add") + } + + return ntotal, nil +} + +// Search returns search result as []SearchResult. +func (f *faiss) Search(k, nq int, xq []float32) ([]SearchResult, error) { + if len(xq) != nq*int(f.dimension) { + return nil, errors.ErrIncompatibleDimensionSize(len(xq), int(f.dimension)) + } + + I := make([]int64, k*nq) + D := make([]float32, k*nq) + f.mu.RLock() + ret := C.faiss_search(f.st, (C.int)(k), (C.int)(nq), (*C.float)(&xq[0]), (*C.long)(&I[0]), (*C.float)(&D[0])) + f.mu.RUnlock() + if ret == ErrorCode { + return nil, errors.NewFaissError("failed to faiss_search") + } + + if len(I) == 0 || len(D) == 0 { + return nil, errors.ErrEmptySearchResult + } + + result := make([]SearchResult, k) + for i := range result { + result[i] = SearchResult{uint32(I[i]), D[i], nil} + } + + return result, nil +} + +// Remove removes from faiss index. +func (f *faiss) Remove(size int, ids []int64) (int, error) { + f.mu.Lock() + ntotal := int(C.faiss_remove(f.st, (C.int)(size), (*C.long)(&ids[0]))) + f.mu.Unlock() + if ntotal < 0 { + return ntotal, errors.NewFaissError("failed to faiss_remove") + } + + return ntotal, nil +} + +// Close faiss index. +func (f *faiss) Close() { + if f.st != nil { + C.faiss_free(f.st) + f.st = nil + } +} diff --git a/internal/core/algorithm/faiss/option.go b/internal/core/algorithm/faiss/option.go new file mode 100644 index 00000000000..40c9a3ad5a0 --- /dev/null +++ b/internal/core/algorithm/faiss/option.go @@ -0,0 +1,120 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package faiss provides implementation of Go API for https://github.com/facebookresearch/faiss +package faiss + +import "C" + +import ( + "strings" + + "github.com/vdaas/vald/internal/core/algorithm" + "github.com/vdaas/vald/internal/errors" +) + +// Option represents the functional option for faiss. +type Option func(*faiss) error + +var defaultOptions = []Option{ + WithDimension(64), + WithNlist(100), + WithM(8), + WithNbitsPerIdx(8), + WithMetricType("l2"), +} + +// WithDimension represents the option to set the dimension for faiss. +func WithDimension(dim int) Option { + return func(f *faiss) error { + if dim > algorithm.MaximumVectorDimensionSize || dim < algorithm.MinimumVectorDimensionSize { + err := errors.ErrInvalidDimensionSize(dim, algorithm.MaximumVectorDimensionSize) + return errors.NewErrCriticalOption("dimension", dim, err) + } + + f.dimension = (C.int)(dim) + return nil + } +} + +// WithNlist represents the option to set the nlist for faiss. +func WithNlist(nlist int) Option { + return func(f *faiss) error { + if nlist <= 0 { + return errors.NewErrInvalidOption("nlist", nlist) + } + + f.nlist = (C.int)(nlist) + return nil + } +} + +// WithM represents the option to set the m for faiss. +func WithM(m int) Option { + return func(f *faiss) error { + if m <= 0 || int(f.dimension)%m != 0 { + return errors.NewErrInvalidOption("m", m) + } + + f.m = (C.int)(m) + return nil + } +} + +// WithNbitsPerIdx represents the option to set the n bits per index for faiss. +func WithNbitsPerIdx(nbitsPerIdx int) Option { + return func(f *faiss) error { + if nbitsPerIdx <= 0 { + return errors.NewErrInvalidOption("nbitsPerIdx", nbitsPerIdx) + } + + f.nbitsPerIdx = (C.int)(nbitsPerIdx) + return nil + } +} + +// WithMetricType represents the option to set the metric type for faiss. +func WithMetricType(metricType string) Option { + return func(f *faiss) error { + if len(metricType) == 0 { + return errors.NewErrIgnoredOption("metricType") + } + + switch strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(metricType)) { + case "innerproduct": + f.metricType = InnerProduct + case "l2": + f.metricType = L2 + default: + err := errors.ErrUnsupportedDistanceType + return errors.NewErrCriticalOption("metricType", metricType, err) + } + + return nil + } +} + +// WithIndexPath represents the option to set the index path for faiss. +func WithIndexPath(idxPath string) Option { + return func(f *faiss) error { + if len(idxPath) == 0 { + return errors.NewErrIgnoredOption("indexPath") + } + + f.idxPath = idxPath + return nil + } +} diff --git a/internal/errgroup/group_test.go b/internal/errgroup/group_test.go index 776f9986127..df88046904d 100644 --- a/internal/errgroup/group_test.go +++ b/internal/errgroup/group_test.go @@ -379,8 +379,7 @@ func Test_group_Limitation(t *testing.T) { }, want: want{ want: &group{ - enableLimitation: func() atomic.Bool { - var el atomic.Bool + enableLimitation: func() (el atomic.Bool) { el.Store(false) return el }(), @@ -398,8 +397,7 @@ func Test_group_Limitation(t *testing.T) { }, want: want{ want: &group{ - enableLimitation: func() atomic.Bool { - var el atomic.Bool + enableLimitation: func() (el atomic.Bool) { el.Store(true) return el }(), diff --git a/internal/errors/faiss.go b/internal/errors/faiss.go new file mode 100644 index 00000000000..735942079c9 --- /dev/null +++ b/internal/errors/faiss.go @@ -0,0 +1,32 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package errors provides error types and function +package errors + +type FaissError struct { + Msg string +} + +func NewFaissError(msg string) error { + return FaissError{ + Msg: msg, + } +} + +func (f FaissError) Error() string { + return f.Msg +} diff --git a/internal/info/info.go b/internal/info/info.go index 7baeae5d6e5..abf2870e19e 100644 --- a/internal/info/info.go +++ b/internal/info/info.go @@ -57,7 +57,7 @@ type Detail struct { GoArch string `json:"go_arch,omitempty" yaml:"go_arch,omitempty"` GoRoot string `json:"go_root,omitempty" yaml:"go_root,omitempty"` CGOEnabled string `json:"cgo_enabled,omitempty" yaml:"cgo_enabled,omitempty"` - NGTVersion string `json:"ngt_version,omitempty" yaml:"ngt_version,omitempty"` + AlgorithmInfo string `json:"algorithm_info,omitempty" yaml:"algorithm_info,omitempty"` BuildCPUInfoFlags []string `json:"build_cpu_info_flags,omitempty" yaml:"build_cpu_info_flags,omitempty"` StackTrace []StackTrace `json:"stack_trace,omitempty" yaml:"stack_trace,omitempty"` } @@ -89,8 +89,8 @@ var ( GoRoot string // CGOEnabled represent the cgo is enable or not to build Vald. CGOEnabled string - // NGTVersion represent the NGT version in Vald. - NGTVersion string + // AlgorithmInfo represent the NGT version in Vald. + AlgorithmInfo string // BuildCPUInfoFlags represent the CPU info flags to build Vald. BuildCPUInfoFlags string @@ -146,7 +146,7 @@ func New(opts ...Option) (Info, error) { GoArch: GoArch, GoRoot: GoRoot, CGOEnabled: CGOEnabled, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -361,8 +361,8 @@ func (i *info) prepare() { default: i.detail.CGOEnabled = cgoUnknown } - if len(i.detail.NGTVersion) == 0 && len(NGTVersion) != 0 { - i.detail.NGTVersion = NGTVersion + if len(i.detail.AlgorithmInfo) == 0 && len(AlgorithmInfo) != 0 { + i.detail.AlgorithmInfo = AlgorithmInfo } if len(i.detail.BuildCPUInfoFlags) == 0 && len(BuildCPUInfoFlags) != 0 { i.detail.BuildCPUInfoFlags = strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " ") diff --git a/internal/info/info_test.go b/internal/info/info_test.go index 0bb08f61eec..2bd189939b0 100644 --- a/internal/info/info_test.go +++ b/internal/info/info_test.go @@ -96,7 +96,7 @@ func TestString(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: "", + AlgorithmInfo: "", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -128,7 +128,7 @@ func TestString(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: "", + AlgorithmInfo: "", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -200,7 +200,7 @@ func TestGet(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: "", + AlgorithmInfo: "", BuildCPUInfoFlags: []string{""}, StackTrace: make([]StackTrace, 0, 10), }, @@ -270,7 +270,7 @@ func TestInit(t *testing.T) { version := Version buildTime := BuildTime cgoEnabled := CGOEnabled - ngtVersion := NGTVersion + ngtVersion := AlgorithmInfo buildCPUInfoFlags := BuildCPUInfoFlags tests := []test{ { @@ -282,16 +282,16 @@ func TestInit(t *testing.T) { want: &info{ baseURL: "https://github.com/vdaas/vald/tree/gitcommit", detail: Detail{ - GitCommit: "gitcommit", - ServerName: "gateway", - Version: "gitcommit", - BuildTime: "1s", - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoTrue, - NGTVersion: "v1.11.6", + GitCommit: "gitcommit", + ServerName: "gateway", + Version: "gitcommit", + BuildTime: "1s", + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoTrue, + AlgorithmInfo: "v1.11.6", BuildCPUInfoFlags: []string{ "avx512f", "avx512dq", }, @@ -311,7 +311,7 @@ func TestInit(t *testing.T) { Version = "" BuildTime = "1s" CGOEnabled = cgoTrue - NGTVersion = "v1.11.6" + AlgorithmInfo = "v1.11.6" BuildCPUInfoFlags = "\t\tavx512f avx512dq\t" }, afterFunc: func(t *testing.T, _ args) { @@ -323,7 +323,7 @@ func TestInit(t *testing.T) { Version = version BuildTime = buildTime CGOEnabled = cgoEnabled - NGTVersion = ngtVersion + AlgorithmInfo = ngtVersion BuildCPUInfoFlags = buildCPUInfoFlags }, }, @@ -336,16 +336,16 @@ func TestInit(t *testing.T) { want: &info{ baseURL: "https://github.com/vdaas/vald/tree/gitcommit", detail: Detail{ - GitCommit: "gitcommit", - ServerName: "", - Version: "gitcommit", - BuildTime: "1s", - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoTrue, - NGTVersion: "v1.11.6", + GitCommit: "gitcommit", + ServerName: "", + Version: "gitcommit", + BuildTime: "1s", + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoTrue, + AlgorithmInfo: "v1.11.6", BuildCPUInfoFlags: []string{ "avx512f", "avx512dq", }, @@ -365,7 +365,7 @@ func TestInit(t *testing.T) { Version = "" BuildTime = "1s" CGOEnabled = cgoTrue - NGTVersion = "v1.11.6" + AlgorithmInfo = "v1.11.6" BuildCPUInfoFlags = "\t\tavx512f avx512dq\t" }, afterFunc: func(t *testing.T, _ args) { @@ -377,7 +377,7 @@ func TestInit(t *testing.T) { Version = version BuildTime = buildTime CGOEnabled = cgoEnabled - NGTVersion = ngtVersion + AlgorithmInfo = ngtVersion BuildCPUInfoFlags = buildCPUInfoFlags }, }, @@ -463,7 +463,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -496,7 +496,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), }, prepOnce: func() (o sync.Once) { @@ -532,7 +532,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -567,7 +567,7 @@ func TestNew(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: strings.Split(strings.TrimSpace(BuildCPUInfoFlags), " "), StackTrace: nil, }, @@ -666,7 +666,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -689,7 +689,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -715,7 +715,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{}, }, @@ -734,7 +734,7 @@ func Test_info_String(t *testing.T) { GoArch: "goarch", GoRoot: "/usr/local/go", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -780,7 +780,7 @@ func TestDetail_String(t *testing.T) { GoOS string GoArch string CGOEnabled string - NGTVersion string + AlgorithmInfo string BuildCPUInfoFlags []string StackTrace []StackTrace } @@ -824,7 +824,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -845,7 +845,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{ { @@ -869,7 +869,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: []StackTrace{}, }, @@ -883,7 +883,7 @@ func TestDetail_String(t *testing.T) { GoOS: "goos", GoArch: "goarch", CGOEnabled: cgoTrue, - NGTVersion: "1.2", + AlgorithmInfo: "1.2", BuildCPUInfoFlags: nil, StackTrace: nil, }, @@ -914,7 +914,7 @@ func TestDetail_String(t *testing.T) { GoOS: test.fields.GoOS, GoArch: test.fields.GoArch, CGOEnabled: test.fields.CGOEnabled, - NGTVersion: test.fields.NGTVersion, + AlgorithmInfo: test.fields.AlgorithmInfo, BuildCPUInfoFlags: test.fields.BuildCPUInfoFlags, StackTrace: test.fields.StackTrace, } @@ -960,17 +960,17 @@ func Test_info_Get(t *testing.T) { }, want: want{ want: Detail{ - ServerName: "", - Version: "", - GitCommit: GitCommit, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - StackTrace: []StackTrace{}, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + ServerName: "", + Version: "", + GitCommit: GitCommit, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + StackTrace: []StackTrace{}, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1015,8 +1015,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1061,8 +1061,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1106,8 +1106,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1151,8 +1151,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1196,8 +1196,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1241,8 +1241,8 @@ func Test_info_Get(t *testing.T) { Line: 100, }, }, - NGTVersion: NGTVersion, - BuildTime: BuildTime, + AlgorithmInfo: AlgorithmInfo, + BuildTime: BuildTime, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1318,15 +1318,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1348,15 +1348,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/internal", detail: Detail{ - GitCommit: "internal", - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: "internal", + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1378,15 +1378,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "v1.0.0", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "v1.0.0", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1408,15 +1408,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: "10s", - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: "10s", + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1438,15 +1438,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: "1.14", - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: "1.14", + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1468,15 +1468,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: "linux", - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: "linux", + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1498,15 +1498,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: "amd", - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: "amd", + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1528,15 +1528,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoTrue, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoTrue, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1558,15 +1558,15 @@ func Test_info_prepare(t *testing.T) { want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoFalse, - NGTVersion: NGTVersion, + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoFalse, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1578,25 +1578,25 @@ func Test_info_prepare(t *testing.T) { }, }, { - name: "set success with NGTVersion set", + name: "set success with AlgorithmInfo set", fields: fields{ detail: Detail{ - NGTVersion: "v1.11.5", + AlgorithmInfo: "v1.11.5", }, }, want: want{ want: info{ baseURL: "https://github.com/vdaas/vald/tree/main", detail: Detail{ - GitCommit: GitCommit, - Version: "", - BuildTime: BuildTime, - GoVersion: runtime.Version(), - GoOS: runtime.GOOS, - GoArch: runtime.GOARCH, - GoRoot: runtime.GOROOT(), - CGOEnabled: cgoUnknown, - NGTVersion: "v1.11.5", + GitCommit: GitCommit, + Version: "", + BuildTime: BuildTime, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + GoRoot: runtime.GOROOT(), + CGOEnabled: cgoUnknown, + AlgorithmInfo: "v1.11.5", BuildCPUInfoFlags: func() []string { if len(BuildCPUInfoFlags) == 0 { return nil @@ -1626,7 +1626,7 @@ func Test_info_prepare(t *testing.T) { GoArch: runtime.GOARCH, GoRoot: runtime.GOROOT(), CGOEnabled: cgoUnknown, - NGTVersion: NGTVersion, + AlgorithmInfo: AlgorithmInfo, BuildCPUInfoFlags: []string{"avx512f"}, }, }, diff --git a/internal/net/dialer.go b/internal/net/dialer.go index 39251a9c7bf..b6882ba85a7 100644 --- a/internal/net/dialer.go +++ b/internal/net/dialer.go @@ -172,7 +172,7 @@ func (d *dialer) lookup(ctx context.Context, host string) (dc *dialerCache, err dc = &dialerCache{ ips: ips, } - log.Infof("lookup succeed %v", dc.ips) + log.Debugf("lookup succeed for %s, ips: %v", host, dc.ips) if d.enableDNSCache { d.dnsCache.Set(host, dc) } diff --git a/internal/net/grpc/admin/admin.go b/internal/net/grpc/admin/admin.go new file mode 100644 index 00000000000..bdf3e99719f --- /dev/null +++ b/internal/net/grpc/admin/admin.go @@ -0,0 +1,24 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package admin provides grpc admin metrics registration API for providing grpc metrics endpoints +package admin + +import ( + "google.golang.org/grpc/admin" +) + +var Register = admin.Register diff --git a/internal/net/grpc/errdetails/errdetails.go b/internal/net/grpc/errdetails/errdetails.go index 24eeaa19c24..c48b552089e 100644 --- a/internal/net/grpc/errdetails/errdetails.go +++ b/internal/net/grpc/errdetails/errdetails.go @@ -282,7 +282,7 @@ func DebugInfoFromInfoDetail(v *info.Detail) *DebugInfo { "Name:", v.ServerName, ",", "GitCommit:", v.GitCommit, ",", "BuildTime:", v.BuildTime, ",", - "NGT_Version:", v.NGTVersion, ",", + "Algorithm_Info:", v.AlgorithmInfo, ",", "Go_Version:", v.GoVersion, ",", "GOARCH:", v.GoArch, ",", "GOOS:", v.GoOS, ",", diff --git a/internal/net/grpc/interceptor/server/logging/accesslog.go b/internal/net/grpc/interceptor/server/logging/accesslog.go index 8fd8be7a6ec..17ab7cf39d8 100644 --- a/internal/net/grpc/interceptor/server/logging/accesslog.go +++ b/internal/net/grpc/interceptor/server/logging/accesslog.go @@ -88,9 +88,9 @@ func AccessLogInterceptor() grpc.UnaryServerInterceptor { if err != nil { entity.Error = err - log.Infod(rpcCompletedMessage, entity) + log.Warn(rpcCompletedMessage, entity) } else { - log.Infod(rpcCompletedMessage, entity) + log.Debug(rpcCompletedMessage, entity) } return resp, err @@ -136,9 +136,9 @@ func AccessLogStreamInterceptor() grpc.StreamServerInterceptor { if err != nil { entity.Error = err - log.Infod(rpcCompletedMessage, entity) + log.Warn(rpcCompletedMessage, entity) } else { - log.Infod(rpcCompletedMessage, entity) + log.Debug(rpcCompletedMessage, entity) } return err diff --git a/internal/net/grpc/interceptor/server/metric/metric.go b/internal/net/grpc/interceptor/server/metric/metric.go index f7a2e7cb81f..02490641c0b 100644 --- a/internal/net/grpc/interceptor/server/metric/metric.go +++ b/internal/net/grpc/interceptor/server/metric/metric.go @@ -36,7 +36,7 @@ const ( func MetricInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor, error) { meter := metrics.GetMeter() - latencyHistgram, err := meter.SyncFloat64().Histogram( + latencyHistgram, err := meter.Float64Histogram( latencyMetricsName, metrics.WithDescription("Server latency in milliseconds, by method"), metrics.WithUnit(metrics.Milliseconds), @@ -45,7 +45,7 @@ func MetricInterceptors() (grpc.UnaryServerInterceptor, grpc.StreamServerInterce return nil, nil, errors.Wrap(err, "failed to create latency metric") } - completedRPCCnt, err := meter.SyncInt64().Counter( + completedRPCCnt, err := meter.Int64Counter( completedRPCsMetricsName, metrics.WithDescription("Count of RPCs by method and status"), metrics.WithUnit(metrics.Milliseconds), diff --git a/internal/observability/exporter/otlp/otlp.go b/internal/observability/exporter/otlp/otlp.go index a0c2c96285f..de722624f86 100644 --- a/internal/observability/exporter/otlp/otlp.go +++ b/internal/observability/exporter/otlp/otlp.go @@ -27,12 +27,11 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/metric/global" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.12.0" + semconv "go.opentelemetry.io/otel/semconv/v1.20.0" ) // Metrics and Trace attribute keys. @@ -123,7 +122,8 @@ func (e *exp) initMeter(ctx context.Context) (err error) { e.metricsExporter, metric.WithInterval(e.mExportInterval), metric.WithTimeout(e.mExportTimeout), - ), e.metricsViews...), + )), + metric.WithView(e.metricsViews...), metric.WithResource(resource.NewWithAttributes( semconv.SchemaURL, e.attributes..., @@ -144,7 +144,7 @@ func (e *exp) Start(ctx context.Context) error { otel.SetTextMapPropagator( propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}), ) - global.SetMeterProvider(e.meterProvider) + otel.SetMeterProvider(e.meterProvider) return nil } diff --git a/internal/observability/metrics/agent/core/faiss/faiss.go b/internal/observability/metrics/agent/core/faiss/faiss.go new file mode 100644 index 00000000000..fb0a68c34c2 --- /dev/null +++ b/internal/observability/metrics/agent/core/faiss/faiss.go @@ -0,0 +1,267 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package faiss + +import ( + "context" + + "github.com/vdaas/vald/internal/observability/metrics" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/aggregation" +) + +const ( + indexCountMetricsName = "agent_core_faiss_index_count" + indexCountMetricsDescription = "Agent Faiss index count" + + uncommittedIndexCountMetricsName = "agent_core_faiss_uncommitted_index_count" + uncommittedIndexCountMetricsDescription = "Agent Faiss index count" + + insertVQueueCountMetricsName = "agent_core_faiss_insert_vqueue_count" + insertVQueueCountMetricsDescription = "Agent Faiss insert vqueue count" + + deleteVQueueCountMetricsName = "agent_core_faiss_delete_vqueue_count" + deleteVQueueCountMetricsDescription = "Agent Faiss delete vqueue count" + + completedCreateIndexTotalMetricsName = "agent_core_faiss_completed_create_index_total" + completedCreateIndexTotalMetricsDescription = "The cumulative count of completed create index execution" + + executedProactiveGCTotalMetricsName = "agent_core_faiss_executed_proactive_gc_total" + executedProactiveGCTotalMetricsDescription = "The cumulative count of proactive GC execution" + + isIndexingMetricsName = "agent_core_faiss_is_indexing" + isIndexingMetricsDescription = "Currently indexing or no" + + isSavingMetricsName = "agent_core_faiss_is_saving" + isSavingMetricsDescription = "Currently saving or not" + + trainCountMetricsName = "agent_core_faiss_train_count" + trainCountMetricsDescription = "Agent Faiss train count" +) + +type faissMetrics struct { + faiss service.Faiss +} + +func New(f service.Faiss) metrics.Metric { + return &faissMetrics{ + faiss: f, + } +} + +func (f *faissMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: indexCountMetricsName, + Description: indexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uncommittedIndexCountMetricsName, + Description: uncommittedIndexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: insertVQueueCountMetricsName, + Description: insertVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: deleteVQueueCountMetricsName, + Description: deleteVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: completedCreateIndexTotalMetricsName, + Description: completedCreateIndexTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: executedProactiveGCTotalMetricsName, + Description: executedProactiveGCTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isIndexingMetricsName, + Description: isIndexingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isSavingMetricsName, + Description: isSavingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: trainCountMetricsName, + Description: trainCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + }, nil +} + +func (f *faissMetrics) Register(m metrics.Meter) error { + indexCount, err := m.Int64ObservableGauge( + indexCountMetricsName, + metrics.WithDescription(indexCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + uncommittedIndexCount, err := m.Int64ObservableGauge( + uncommittedIndexCountMetricsName, + metrics.WithDescription(uncommittedIndexCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + insertVQueueCount, err := m.Int64ObservableGauge( + insertVQueueCountMetricsName, + metrics.WithDescription(insertVQueueCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + deleteVQueueCount, err := m.Int64ObservableGauge( + deleteVQueueCountMetricsName, + metrics.WithDescription(deleteVQueueCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + completedCreateIndexTotal, err := m.Int64ObservableGauge( + completedCreateIndexTotalMetricsName, + metrics.WithDescription(completedCreateIndexTotalMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + executedProactiveGCTotal, err := m.Int64ObservableGauge( + executedProactiveGCTotalMetricsName, + metrics.WithDescription(executedProactiveGCTotalMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + isIndexing, err := m.Int64ObservableGauge( + isIndexingMetricsName, + metrics.WithDescription(isIndexingMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + isSaving, err := m.Int64ObservableGauge( + isSavingMetricsName, + metrics.WithDescription(isSavingMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + trainCount, err := m.Int64ObservableGauge( + trainCountMetricsName, + metrics.WithDescription(trainCountMetricsDescription), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { + var indexing int64 + if f.faiss.IsIndexing() { + indexing = 1 + } + var saving int64 + if f.faiss.IsSaving() { + saving = 1 + } + + o.ObserveInt64(indexCount, int64(f.faiss.Len())) + o.ObserveInt64(uncommittedIndexCount, int64(f.faiss.InsertVQueueBufferLen()+f.faiss.DeleteVQueueBufferLen())) + o.ObserveInt64(insertVQueueCount, int64(f.faiss.InsertVQueueBufferLen())) + o.ObserveInt64(deleteVQueueCount, int64(int64(f.faiss.DeleteVQueueBufferLen()))) + o.ObserveInt64(completedCreateIndexTotal, int64(f.faiss.NumberOfCreateIndexExecution())) + o.ObserveInt64(executedProactiveGCTotal, int64(f.faiss.NumberOfProactiveGCExecution())) + o.ObserveInt64(isIndexing, int64(indexing)) + o.ObserveInt64(isSaving, int64(saving)) + o.ObserveInt64(trainCount, int64(f.faiss.GetTrainSize())) + + return nil + }, + indexCount, + uncommittedIndexCount, + insertVQueueCount, + deleteVQueueCount, + completedCreateIndexTotal, + executedProactiveGCTotal, + isIndexing, + isSaving, + trainCount, + ) + return err +} diff --git a/internal/observability/metrics/agent/core/ngt/ngt.go b/internal/observability/metrics/agent/core/ngt/ngt.go index 89e927ef52b..7cfa503c721 100644 --- a/internal/observability/metrics/agent/core/ngt/ngt.go +++ b/internal/observability/metrics/agent/core/ngt/ngt.go @@ -18,8 +18,9 @@ import ( "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/pkg/agent/core/ngt/service" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -61,103 +62,85 @@ func New(n service.NGT) metrics.Metric { } } -func (n *ngtMetrics) View() ([]*metrics.View, error) { - indexCount, err := view.New( - view.MatchInstrumentName(indexCountMetricsName), - view.WithSetDescription(indexCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - uncommittedIndexCount, err := view.New( - view.MatchInstrumentName(uncommittedIndexCountMetricsName), - view.WithSetDescription(uncommittedIndexCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - insertVQueueCount, err := view.New( - view.MatchInstrumentName(insertVQueueCountMetricsName), - view.WithSetDescription(insertVQueueCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - deleteVQueueCount, err := view.New( - view.MatchInstrumentName(deleteVQueueCountMetricsName), - view.WithSetDescription(deleteVQueueCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - completedCreateIndexTotal, err := view.New( - view.MatchInstrumentName(completedCreateIndexTotalMetricsName), - view.WithSetDescription(completedCreateIndexTotalMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - executedProactiveGCTotal, err := view.New( - view.MatchInstrumentName(executedProactiveGCTotalMetricsName), - view.WithSetDescription(executedProactiveGCTotalMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - isIndexing, err := view.New( - view.MatchInstrumentName(isIndexingMetricsName), - view.WithSetDescription(isIndexingMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - isSaving, err := view.New( - view.MatchInstrumentName(isSavingMetricsName), - view.WithSetDescription(isSavingMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - brokenIndexCount, err := view.New( - view.MatchInstrumentName(brokenIndexStoreCountMetricsName), - view.WithSetDescription(brokenIndexStoreCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &indexCount, - &uncommittedIndexCount, - &insertVQueueCount, - &deleteVQueueCount, - &completedCreateIndexTotal, - &executedProactiveGCTotal, - &isIndexing, - &isSaving, - &brokenIndexCount, +func (n *ngtMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: indexCountMetricsName, + Description: indexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uncommittedIndexCountMetricsName, + Description: uncommittedIndexCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: insertVQueueCountMetricsName, + Description: insertVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: deleteVQueueCountMetricsName, + Description: deleteVQueueCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: completedCreateIndexTotalMetricsName, + Description: completedCreateIndexTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: executedProactiveGCTotalMetricsName, + Description: executedProactiveGCTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isIndexingMetricsName, + Description: isIndexingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isSavingMetricsName, + Description: isSavingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (n *ngtMetrics) Register(m metrics.Meter) error { - indexCount, err := m.AsyncInt64().Gauge( + indexCount, err := m.Int64ObservableGauge( indexCountMetricsName, metrics.WithDescription(indexCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -166,7 +149,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - uncommittedIndexCount, err := m.AsyncInt64().Gauge( + uncommittedIndexCount, err := m.Int64ObservableGauge( uncommittedIndexCountMetricsName, metrics.WithDescription(uncommittedIndexCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -175,7 +158,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - insertVQueueCount, err := m.AsyncInt64().Gauge( + insertVQueueCount, err := m.Int64ObservableGauge( insertVQueueCountMetricsName, metrics.WithDescription(insertVQueueCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -184,7 +167,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - deleteVQueueCount, err := m.AsyncInt64().Gauge( + deleteVQueueCount, err := m.Int64ObservableGauge( deleteVQueueCountMetricsName, metrics.WithDescription(deleteVQueueCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -193,7 +176,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - completedCreateIndexTotal, err := m.AsyncInt64().Gauge( + completedCreateIndexTotal, err := m.Int64ObservableGauge( completedCreateIndexTotalMetricsName, metrics.WithDescription(completedCreateIndexTotalMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -202,7 +185,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - executedProactiveGCTotal, err := m.AsyncInt64().Gauge( + executedProactiveGCTotal, err := m.Int64ObservableGauge( executedProactiveGCTotalMetricsName, metrics.WithDescription(executedProactiveGCTotalMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -211,7 +194,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - isIndexing, err := m.AsyncInt64().Gauge( + isIndexing, err := m.Int64ObservableGauge( isIndexingMetricsName, metrics.WithDescription(isIndexingMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -220,7 +203,7 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - isSaving, err := m.AsyncInt64().Gauge( + isSaving, err := m.Int64ObservableGauge( isSavingMetricsName, metrics.WithDescription(isSavingMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -229,47 +212,34 @@ func (n *ngtMetrics) Register(m metrics.Meter) error { return err } - brokenIndexCount, err := m.AsyncInt64().Gauge( - brokenIndexStoreCountMetricsName, - metrics.WithDescription(brokenIndexStoreCountMetricsDescription), - metrics.WithUnit(metrics.Dimensionless), - ) - if err != nil { - return err - } - - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - indexCount, - uncommittedIndexCount, - insertVQueueCount, - deleteVQueueCount, - completedCreateIndexTotal, - executedProactiveGCTotal, - isIndexing, - isSaving, - brokenIndexCount, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { var indexing int64 if n.ngt.IsIndexing() { indexing = 1 } - var saving int64 if n.ngt.IsSaving() { saving = 1 } - - indexCount.Observe(ctx, int64(n.ngt.Len())) - uncommittedIndexCount.Observe(ctx, int64(n.ngt.InsertVQueueBufferLen()+n.ngt.DeleteVQueueBufferLen())) - insertVQueueCount.Observe(ctx, int64(n.ngt.InsertVQueueBufferLen())) - deleteVQueueCount.Observe(ctx, int64(int64(n.ngt.DeleteVQueueBufferLen()))) - completedCreateIndexTotal.Observe(ctx, int64(n.ngt.NumberOfCreateIndexExecution())) - executedProactiveGCTotal.Observe(ctx, int64(n.ngt.NumberOfProactiveGCExecution())) - isIndexing.Observe(ctx, int64(indexing)) - isSaving.Observe(ctx, int64(saving)) - brokenIndexCount.Observe(ctx, int64(n.ngt.BrokenIndexCount())) + o.ObserveInt64(indexCount, int64(n.ngt.Len())) + o.ObserveInt64(uncommittedIndexCount, int64(n.ngt.InsertVQueueBufferLen()+n.ngt.DeleteVQueueBufferLen())) + o.ObserveInt64(insertVQueueCount, int64(n.ngt.InsertVQueueBufferLen())) + o.ObserveInt64(deleteVQueueCount, int64(int64(n.ngt.DeleteVQueueBufferLen()))) + o.ObserveInt64(completedCreateIndexTotal, int64(n.ngt.NumberOfCreateIndexExecution())) + o.ObserveInt64(executedProactiveGCTotal, int64(n.ngt.NumberOfProactiveGCExecution())) + o.ObserveInt64(isIndexing, int64(indexing)) + o.ObserveInt64(isSaving, int64(saving)) + return nil }, - ) + indexCount, + uncommittedIndexCount, + insertVQueueCount, + deleteVQueueCount, + completedCreateIndexTotal, + executedProactiveGCTotal, + isIndexing, + isSaving, + ) + return err } diff --git a/internal/observability/metrics/agent/sidecar/sidecar.go b/internal/observability/metrics/agent/sidecar/sidecar.go index 34ec7934ea2..991a010b906 100644 --- a/internal/observability/metrics/agent/sidecar/sidecar.go +++ b/internal/observability/metrics/agent/sidecar/sidecar.go @@ -21,8 +21,9 @@ import ( "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/pkg/agent/sidecar/service/observer" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -58,45 +59,42 @@ func New() MetricsHook { } } -func (*sidecarMetrics) View() ([]*metrics.View, error) { - uploadTotal, err := view.New( - view.MatchInstrumentName(uploadTotalMetricsName), - view.WithSetDescription(uploadTotalMetricsDescription), - view.WithSetAggregation(aggregation.Sum{}), - ) - if err != nil { - return nil, err - } - - uploadBytes, err := view.New( - view.MatchInstrumentName(uploadBytesMetricsName), - view.WithSetDescription(uploadBytesMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - uploadLatency, err := view.New( - view.MatchInstrumentName(uploadLatencyMetricsName), - view.WithSetDescription(uploadLatencyMetricsDescription), - view.WithSetAggregation(aggregation.ExplicitBucketHistogram{ - Boundaries: metrics.RoughMillisecondsDistribution, - }), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &uploadTotal, - &uploadBytes, - &uploadLatency, +func (*sidecarMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: uploadTotalMetricsName, + Description: uploadTotalMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.Sum{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uploadBytesMetricsName, + Description: uploadBytesMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uploadLatencyMetricsName, + Description: uploadLatencyMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.ExplicitBucketHistogram{ + Boundaries: metrics.RoughMillisecondsDistribution, + }, + }, + ), }, nil } func (sm *sidecarMetrics) Register(m metrics.Meter) error { - uploadTotal, err := m.AsyncInt64().Counter( + uploadTotal, err := m.Int64ObservableCounter( uploadTotalMetricsName, metrics.WithDescription(uploadTotalMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -104,7 +102,7 @@ func (sm *sidecarMetrics) Register(m metrics.Meter) error { if err != nil { return err } - uploadBytes, err := m.AsyncInt64().Gauge( + uploadBytes, err := m.Int64ObservableGauge( uploadBytesMetricsName, metrics.WithDescription(uploadBytesMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -112,7 +110,7 @@ func (sm *sidecarMetrics) Register(m metrics.Meter) error { if err != nil { return err } - uploadLatency, err := m.AsyncFloat64().Gauge( + uploadLatency, err := m.Float64ObservableGauge( uploadLatencyMetricsName, metrics.WithDescription(uploadLatencyMetricsDescription), metrics.WithUnit(metrics.Milliseconds), @@ -121,18 +119,13 @@ func (sm *sidecarMetrics) Register(m metrics.Meter) error { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - uploadTotal, - uploadBytes, - uploadLatency, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { sm.mu.Lock() defer sm.mu.Unlock() if sm.info == nil { - return + return nil } attrs := []attribute.KeyValue{ @@ -141,15 +134,19 @@ func (sm *sidecarMetrics) Register(m metrics.Meter) error { attribute.String(sm.filenameKey, sm.info.Filename), } + o.ObserveInt64(uploadTotal, 1, attrs...) + o.ObserveInt64(uploadBytes, sm.info.Bytes, attrs...) latencyMillis := float64(sm.info.EndTime.Sub(sm.info.StartTime)) / float64(time.Millisecond) - - uploadTotal.Observe(ctx, 1, attrs...) - uploadBytes.Observe(ctx, sm.info.Bytes, attrs...) - uploadLatency.Observe(ctx, latencyMillis, attrs...) - + o.ObserveFloat64(uploadLatency, latencyMillis, attrs...) sm.info = nil + + return nil }, + uploadTotal, + uploadBytes, + uploadLatency, ) + return err } func (*sidecarMetrics) BeforeProcess(ctx context.Context, _ *observer.BackupInfo) (context.Context, error) { diff --git a/internal/observability/metrics/backoff/backoff.go b/internal/observability/metrics/backoff/backoff.go index 6d721f831a3..8767cb42dcf 100644 --- a/internal/observability/metrics/backoff/backoff.go +++ b/internal/observability/metrics/backoff/backoff.go @@ -19,8 +19,9 @@ import ( "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -38,22 +39,22 @@ func New() metrics.Metric { } } -func (*backoffMetrics) View() ([]*metrics.View, error) { - retryCount, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &retryCount, +func (*backoffMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } -func (bm *backoffMetrics) Register(m metrics.Meter) error { - retryCount, err := m.AsyncInt64().Gauge( +func (bm *backoffMetrics) Register(m metrics.Meter) (err error) { + retryCount, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -61,18 +62,17 @@ func (bm *backoffMetrics) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - retryCount, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(ctx context.Context, o api.Observer) error { ms := backoff.Metrics(ctx) if len(ms) == 0 { - return + return nil } for name, cnt := range ms { - retryCount.Observe(ctx, cnt, attribute.String(bm.backoffNameKey, name)) + o.ObserveInt64(retryCount, cnt, attribute.String(bm.backoffNameKey, name)) } - }, + return nil + }, retryCount, ) + return err } diff --git a/internal/observability/metrics/circuitbreaker/circuitbreaker.go b/internal/observability/metrics/circuitbreaker/circuitbreaker.go index 9d1cf46084b..d9119a2d154 100644 --- a/internal/observability/metrics/circuitbreaker/circuitbreaker.go +++ b/internal/observability/metrics/circuitbreaker/circuitbreaker.go @@ -19,8 +19,9 @@ import ( "github.com/vdaas/vald/internal/circuitbreaker" "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -40,23 +41,22 @@ func New() metrics.Metric { } } -func (*breakerMetrics) View() ([]*metrics.View, error) { - breakerState, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &breakerState, +func (*breakerMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (bm *breakerMetrics) Register(m metrics.Meter) error { - breakerState, err := m.AsyncInt64().Gauge( + breakerState, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -65,23 +65,22 @@ func (bm *breakerMetrics) Register(m metrics.Meter) error { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - breakerState, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(ctx context.Context, o api.Observer) error { ms := circuitbreaker.Metrics(ctx) if len(ms) != 0 { for name, sts := range ms { if len(sts) != 0 { for st, cnt := range sts { - breakerState.Observe(ctx, cnt, + o.ObserveInt64(breakerState, cnt, attribute.String(bm.breakerNameKey, name), attribute.String(bm.stateKey, st.String())) } } } } - }, + return nil + }, breakerState, ) + return err } diff --git a/internal/observability/metrics/grpc/grpc.go b/internal/observability/metrics/grpc/grpc.go index 2655eda3b9f..0c8f77d7b0a 100644 --- a/internal/observability/metrics/grpc/grpc.go +++ b/internal/observability/metrics/grpc/grpc.go @@ -15,8 +15,8 @@ package grpc import ( "github.com/vdaas/vald/internal/observability/metrics" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -33,29 +33,28 @@ func New() metrics.Metric { return &grpcServerMetrics{} } -func (*grpcServerMetrics) View() ([]*metrics.View, error) { - latencyHistgram, err := view.New( - view.MatchInstrumentName(latencyMetricsName), - view.WithSetDescription(latencyMetricsDesctiption), - view.WithSetAggregation(aggregation.ExplicitBucketHistogram{ - Boundaries: metrics.DefaultMillisecondsDistribution, - }), - ) - if err != nil { - return nil, err - } - - completedRPCCnt, err := view.New( - view.MatchInstrumentName(completedRPCsMetricsName), - view.WithSetDescription(completedRPCsMetricsDescription), - view.WithSetAggregation(aggregation.Sum{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &latencyHistgram, - &completedRPCCnt, +func (*grpcServerMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: latencyMetricsName, + Description: latencyMetricsDesctiption, + }, + view.Stream{ + Aggregation: aggregation.ExplicitBucketHistogram{ + Boundaries: metrics.DefaultMillisecondsDistribution, + }, + }, + ), + view.NewView( + view.Instrument{ + Name: completedRPCsMetricsName, + Description: completedRPCsMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.Sum{}, + }, + ), }, nil } diff --git a/internal/observability/metrics/info/info.go b/internal/observability/metrics/info/info.go index 705af885746..26a4944a718 100644 --- a/internal/observability/metrics/info/info.go +++ b/internal/observability/metrics/info/info.go @@ -21,8 +21,9 @@ import ( "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) type info struct { @@ -83,23 +84,22 @@ func labelKVs(i interface{}) map[string]string { return kvs } -func (i *info) View() ([]*metrics.View, error) { - info, err := view.New( - view.MatchInstrumentName(i.name), - view.WithSetDescription(i.description), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &info, +func (i *info) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: i.name, + Description: i.description, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (i *info) Register(m metrics.Meter) error { - info, err := m.AsyncInt64().Gauge( + info, err := m.Int64ObservableGauge( i.name, metrics.WithDescription(i.description), metrics.WithUnit(metrics.Dimensionless), @@ -107,16 +107,15 @@ func (i *info) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - info, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { attrs := make([]attribute.KeyValue, 0, len(i.kvs)) for key, val := range i.kvs { attrs = append(attrs, attribute.String(key, val)) } - info.Observe(ctx, 1, attrs...) - }, + o.ObserveInt64(info, 1, attrs...) + return nil + }, info, ) + return err } diff --git a/internal/observability/metrics/manager/index/index.go b/internal/observability/metrics/manager/index/index.go index 7e32d371481..84d578dbade 100644 --- a/internal/observability/metrics/manager/index/index.go +++ b/internal/observability/metrics/manager/index/index.go @@ -18,8 +18,9 @@ import ( "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/pkg/manager/index/service" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -43,43 +44,40 @@ func New(i service.Indexer) metrics.Metric { } } -func (*indexerMetrics) View() ([]*metrics.View, error) { - uuidCount, err := view.New( - view.MatchInstrumentName(uuidCountMetricsName), - view.WithSetDescription(uuidCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - uncommittedUUIDCount, err := view.New( - view.MatchInstrumentName(uncommittedUUIDCountMetricsName), - view.WithSetDescription(uncommittedUUIDCountMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - isIndexing, err := view.New( - view.MatchInstrumentName(isIndexingMetricsName), - view.WithSetDescription(isIndexingMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &uuidCount, - &uncommittedUUIDCount, - &isIndexing, +func (*indexerMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: uuidCountMetricsName, + Description: uuidCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: uncommittedUUIDCountMetricsName, + Description: uncommittedUUIDCountMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: isIndexingMetricsName, + Description: isIndexingMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (im *indexerMetrics) Register(m metrics.Meter) error { - uuidCount, err := m.AsyncInt64().Gauge( + uuidCount, err := m.Int64ObservableGauge( uuidCountMetricsName, metrics.WithDescription(uuidCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -88,7 +86,7 @@ func (im *indexerMetrics) Register(m metrics.Meter) error { return err } - uncommittedUUIDCount, err := m.AsyncInt64().Gauge( + uncommittedUUIDCount, err := m.Int64ObservableGauge( uncommittedUUIDCountMetricsName, metrics.WithDescription(uncommittedUUIDCountMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -97,7 +95,7 @@ func (im *indexerMetrics) Register(m metrics.Meter) error { return err } - isIndexing, err := m.AsyncInt64().Gauge( + isIndexing, err := m.Int64ObservableGauge( isIndexingMetricsName, metrics.WithDescription(isIndexingMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -106,20 +104,20 @@ func (im *indexerMetrics) Register(m metrics.Meter) error { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - uuidCount, - uncommittedUUIDCount, - isIndexing, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { var indexing int64 if im.indexer.IsIndexing() { indexing = 1 } - uuidCount.Observe(ctx, int64(im.indexer.NumberOfUUIDs())) - uncommittedUUIDCount.Observe(ctx, int64(im.indexer.NumberOfUncommittedUUIDs())) - isIndexing.Observe(ctx, int64(indexing)) + o.ObserveInt64(uuidCount, int64(im.indexer.NumberOfUUIDs())) + o.ObserveInt64(uncommittedUUIDCount, int64(im.indexer.NumberOfUncommittedUUIDs())) + o.ObserveInt64(isIndexing, int64(indexing)) + return nil }, + uuidCount, + uncommittedUUIDCount, + isIndexing, ) + return nil } diff --git a/internal/observability/metrics/mem/index/index.go b/internal/observability/metrics/mem/index/index.go index 9da794482e4..756f69e5925 100644 --- a/internal/observability/metrics/mem/index/index.go +++ b/internal/observability/metrics/mem/index/index.go @@ -19,8 +19,9 @@ import ( "time" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -73,153 +74,139 @@ func New() metrics.Metric { return &memoryMetrics{} } -func (*memoryMetrics) View() ([]*metrics.View, error) { - alloc, err := view.New( - view.MatchInstrumentName(allocMetricsDescription), - view.WithSetDescription(allocMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - totalAlloc, err := view.New( - view.MatchInstrumentName(totalAllocMetricsDescription), - view.WithSetDescription(totalAllocMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - sys, err := view.New( - view.MatchInstrumentName(sysMetricsName), - view.WithSetDescription(sysMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - mallocs, err := view.New( - view.MatchInstrumentName(mallocsMetricsName), - view.WithSetDescription(mallocsMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - frees, err := view.New( - view.MatchInstrumentName(freesMetricsName), - view.WithSetDescription(freesMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - heapAlloc, err := view.New( - view.MatchInstrumentName(heapAllocMetricsName), - view.WithSetDescription(heapAllocMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - heapSys, err := view.New( - view.MatchInstrumentName(heapSysMetricsName), - view.WithSetDescription(heapSysMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - heapIdle, err := view.New( - view.MatchInstrumentName(heapIdleMetricsName), - view.WithSetDescription(heapIdleMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - heapInuse, err := view.New( - view.MatchInstrumentName(heapInuseMetricsName), - view.WithSetDescription(heapInuseMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - heapReleased, err := view.New( - view.MatchInstrumentName(heapReleasedMetricsName), - view.WithSetDescription(heapReleasedMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - stackInuse, err := view.New( - view.MatchInstrumentName(stackInuseMetricsName), - view.WithSetDescription(stackInuseMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - stackSys, err := view.New( - view.MatchInstrumentName(stackSysMetricsName), - view.WithSetDescription(stackSysMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - pauseTotalMs, err := view.New( - view.MatchInstrumentName(pauseTotalMsMetricsName), - view.WithSetDescription(pauseTotalMsMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - numGC, err := view.New( - view.MatchInstrumentName(numGCMetricsName), - view.WithSetDescription(numGCMetricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &alloc, - &totalAlloc, - &sys, - &mallocs, - &frees, - &heapAlloc, - &heapSys, - &heapIdle, - &heapInuse, - &heapReleased, - &stackInuse, - &stackSys, - &pauseTotalMs, - &numGC, +func (*memoryMetrics) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: allocMetricsName, + Description: allocMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: totalAllocMetricsDescription, + Description: totalAllocMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: sysMetricsName, + Description: sysMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: mallocsMetricsName, + Description: mallocsMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: freesMetricsName, + Description: freesMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapAllocMetricsName, + Description: heapAllocMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapSysMetricsName, + Description: heapSysMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapIdleMetricsName, + Description: heapIdleMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapInuseMetricsName, + Description: heapInuseMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: heapReleasedMetricsName, + Description: heapReleasedMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: stackInuseMetricsName, + Description: stackInuseMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: stackSysMetricsName, + Description: stackSysMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: pauseTotalMsMetricsName, + Description: pauseTotalMsMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), + view.NewView( + view.Instrument{ + Name: numGCMetricsName, + Description: numGCMetricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (*memoryMetrics) Register(m metrics.Meter) error { - alloc, err := m.AsyncInt64().Gauge( + alloc, err := m.Int64ObservableGauge( allocMetricsName, metrics.WithDescription(allocMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -228,7 +215,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - totalAlloc, err := m.AsyncInt64().Gauge( + totalAlloc, err := m.Int64ObservableGauge( totalAllocMetricsDescription, metrics.WithDescription(totalAllocMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -237,7 +224,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - sys, err := m.AsyncInt64().Gauge( + sys, err := m.Int64ObservableGauge( sysMetricsName, metrics.WithDescription(sysMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -246,7 +233,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - mallocs, err := m.AsyncInt64().Gauge( + mallocs, err := m.Int64ObservableGauge( mallocsMetricsName, metrics.WithDescription(mallocsMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -255,7 +242,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - frees, err := m.AsyncInt64().Gauge( + frees, err := m.Int64ObservableGauge( freesMetricsName, metrics.WithDescription(freesMetricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -264,7 +251,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - heapAlloc, err := m.AsyncInt64().Gauge( + heapAlloc, err := m.Int64ObservableGauge( heapAllocMetricsName, metrics.WithDescription(heapAllocMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -273,7 +260,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - heapSys, err := m.AsyncInt64().Gauge( + heapSys, err := m.Int64ObservableGauge( heapSysMetricsName, metrics.WithDescription(heapSysMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -282,7 +269,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - heapIdle, err := m.AsyncInt64().Gauge( + heapIdle, err := m.Int64ObservableGauge( heapIdleMetricsName, metrics.WithDescription(heapIdleMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -291,7 +278,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - heapInuse, err := m.AsyncInt64().Gauge( + heapInuse, err := m.Int64ObservableGauge( heapInuseMetricsName, metrics.WithDescription(heapInuseMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -300,7 +287,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - heapReleased, err := m.AsyncInt64().Gauge( + heapReleased, err := m.Int64ObservableGauge( heapReleasedMetricsName, metrics.WithDescription(heapReleasedMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -309,7 +296,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - stackInuse, err := m.AsyncInt64().Gauge( + stackInuse, err := m.Int64ObservableGauge( stackInuseMetricsName, metrics.WithDescription(stackInuseMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -318,7 +305,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - stackSys, err := m.AsyncInt64().Gauge( + stackSys, err := m.Int64ObservableGauge( stackSysMetricsName, metrics.WithDescription(stackSysMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -327,7 +314,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - pauseTotalMs, err := m.AsyncInt64().Gauge( + pauseTotalMs, err := m.Int64ObservableGauge( pauseTotalMsMetricsName, metrics.WithDescription(pauseTotalMsMetricsDescription), metrics.WithUnit(metrics.Milliseconds), @@ -336,7 +323,7 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - numGC, err := m.AsyncInt64().Gauge( + numGC, err := m.Int64ObservableGauge( numGCMetricsName, metrics.WithDescription(numGCMetricsDescription), metrics.WithUnit(metrics.Bytes), @@ -345,46 +332,44 @@ func (*memoryMetrics) Register(m metrics.Meter) error { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - alloc, - totalAlloc, - sys, - mallocs, - frees, - heapAlloc, - heapSys, - heapIdle, - heapInuse, - heapReleased, - stackInuse, - stackSys, - pauseTotalMs, - numGC, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { var mstats runtime.MemStats runtime.ReadMemStats(&mstats) - - alloc.Observe(ctx, int64(mstats.Alloc)) - totalAlloc.Observe(ctx, int64(mstats.TotalAlloc)) - sys.Observe(ctx, int64(mstats.Sys)) - mallocs.Observe(ctx, int64(mstats.Mallocs)) - frees.Observe(ctx, int64(mstats.Frees)) - heapAlloc.Observe(ctx, int64(mstats.HeapAlloc)) - heapSys.Observe(ctx, int64(mstats.HeapSys)) - heapIdle.Observe(ctx, int64(mstats.HeapIdle)) - heapInuse.Observe(ctx, int64(mstats.HeapInuse)) - heapReleased.Observe(ctx, int64(mstats.HeapReleased)) - stackInuse.Observe(ctx, int64(mstats.StackInuse)) - stackSys.Observe(ctx, int64(mstats.StackSys)) - - ptMs := int64(0) + o.ObserveInt64(alloc, int64(mstats.Alloc)) + o.ObserveInt64(frees, int64(mstats.Frees)) + o.ObserveInt64(heapAlloc, int64(mstats.HeapAlloc)) + o.ObserveInt64(heapIdle, int64(mstats.HeapIdle)) + o.ObserveInt64(heapInuse, int64(mstats.HeapInuse)) + o.ObserveInt64(heapReleased, int64(mstats.HeapReleased)) + o.ObserveInt64(heapSys, int64(mstats.HeapSys)) + o.ObserveInt64(mallocs, int64(mstats.Mallocs)) + o.ObserveInt64(stackInuse, int64(mstats.StackInuse)) + o.ObserveInt64(stackSys, int64(mstats.StackSys)) + o.ObserveInt64(sys, int64(mstats.Sys)) + o.ObserveInt64(totalAlloc, int64(mstats.TotalAlloc)) + var ptMs int64 if mstats.PauseTotalNs > 0 { - ptMs = int64(mstats.PauseTotalNs / uint64(time.Millisecond)) + ptMs = int64(float64(mstats.PauseTotalNs) / float64(time.Millisecond)) } - pauseTotalMs.Observe(ctx, ptMs) - numGC.Observe(ctx, int64(mstats.NextGC)) + o.ObserveInt64(pauseTotalMs, ptMs) + o.ObserveInt64(numGC, int64(mstats.NextGC)) + return nil }, + alloc, + frees, + heapAlloc, + heapIdle, + heapInuse, + heapReleased, + heapSys, + mallocs, + stackInuse, + stackSys, + sys, + totalAlloc, + pauseTotalMs, + numGC, ) + return err } diff --git a/internal/observability/metrics/metrics.go b/internal/observability/metrics/metrics.go index 9e4e35b8679..a819a9518da 100644 --- a/internal/observability/metrics/metrics.go +++ b/internal/observability/metrics/metrics.go @@ -14,16 +14,40 @@ package metrics import ( + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/unit" - "go.opentelemetry.io/otel/sdk/metric/view" + view "go.opentelemetry.io/otel/sdk/metric" ) -const ValdOrg = "vald.vdaas.org" +type ( + View = view.View + + // Meter is type alias of metrics.Meter. + Meter = metric.Meter + // Metric represents an interface for metric. + Metric interface { + View() ([]View, error) + Register(Meter) error + } +) + +const ( + ValdOrg = "vald.vdaas.org" + // Units defined by OpenTelemetry. + // Dimensionless is a type alias of unit.Dimensionless. + Dimensionless = "1" + // Bytes is a type alias of unit.Bytes. + Bytes = "By" + // Milliseconds is a type alias of unit.Milliseconds. + Milliseconds = "ms" +) var ( + // WithUnit returns an metric.WithUnit option. + WithUnit = metric.WithUnit + // WithDescription returns an metric.WithDescription option. + WithDescription = metric.WithDescription + RoughMillisecondsDistribution = []float64{ 1, 5, @@ -111,52 +135,7 @@ var ( } ) -// Meter is type alias of metrics.Meter. -type Meter = metric.Meter - // GetMeter returns the Meter object to record metrics. func GetMeter() Meter { - return global.MeterProvider().Meter(ValdOrg) -} - -// Unit is type alias of unit.Unit. -type Unit = unit.Unit - -// Units defined by OpenTelemetry. -const ( - // Dimensionless is a type alias of unit.Dimensionless. - Dimensionless = unit.Dimensionless - // Bytes is a type alias of unit.Bytes. - Bytes = unit.Bytes - // Milliseconds is a type alias of unit.Milliseconds. - Milliseconds = unit.Milliseconds -) - -type ( - // AsynchronousInstrument is type alias of instrument.Asynchronous. - AsynchronousInstrument = instrument.Asynchronous - // SynchronousInstrument is type alias of instrument.Synchronous. - SynchronousInstrument = instrument.Synchronous -) - -// WithUnit returns an instrument.WithUnit option. -func WithUnit(u Unit) instrument.Option { - return instrument.WithUnit(u) -} - -// WithDescription returns an instrument.WithDescription option. -func WithDescription(desc string) instrument.Option { - return instrument.WithDescription(desc) -} - -type View = view.View - -type Viewer interface { - View() ([]*View, error) -} - -// Metric represents an interface for metric. -type Metric interface { - Viewer - Register(Meter) error + return otel.GetMeterProvider().Meter(ValdOrg) } diff --git a/internal/observability/metrics/metrics_test.go b/internal/observability/metrics/metrics_test.go deleted file mode 100644 index e3f572b6821..00000000000 --- a/internal/observability/metrics/metrics_test.go +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright (C) 2019-2023 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package metrics - -import ( - "reflect" - "testing" - - "github.com/vdaas/vald/internal/errors" - "github.com/vdaas/vald/internal/test/goleak" - "go.opentelemetry.io/otel/metric/instrument" -) - -// NOT IMPLEMENTED BELOW - -func TestGetMeter(t *testing.T) { - type want struct { - want Meter - } - type test struct { - name string - want want - checkFunc func(want, Meter) error - beforeFunc func(*testing.T) - afterFunc func(*testing.T) - } - defaultCheckFunc := func(w want, got Meter) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - want: want{}, - checkFunc: defaultCheckFunc, - beforeFunc: func(t *testing.T,) { - t.Helper() - }, - afterFunc: func(t *testing.T,) { - t.Helper() - }, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - want: want{}, - checkFunc: defaultCheckFunc, - beforeFunc: func(t *testing.T,) { - t.Helper() - }, - afterFunc: func(t *testing.T,) { - t.Helper() - }, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(tt) - } - if test.afterFunc != nil { - defer test.afterFunc(tt) - } - checkFunc := test.checkFunc - if test.checkFunc == nil { - checkFunc = defaultCheckFunc - } - - got := GetMeter() - if err := checkFunc(test.want, got); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func TestWithUnit(t *testing.T) { - type args struct { - u Unit - } - type want struct { - want instrument.Option - } - type test struct { - name string - args args - want want - checkFunc func(want, instrument.Option) error - beforeFunc func(*testing.T, args) - afterFunc func(*testing.T, args) - } - defaultCheckFunc := func(w want, got instrument.Option) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - u:nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - beforeFunc: func(t *testing.T, args args) { - t.Helper() - }, - afterFunc: func(t *testing.T, args args) { - t.Helper() - }, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - u:nil, - }, - want: want{}, - checkFunc: defaultCheckFunc, - beforeFunc: func(t *testing.T, args args) { - t.Helper() - }, - afterFunc: func(t *testing.T, args args) { - t.Helper() - }, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(tt, test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(tt, test.args) - } - checkFunc := test.checkFunc - if test.checkFunc == nil { - checkFunc = defaultCheckFunc - } - - got := WithUnit(test.args.u) - if err := checkFunc(test.want, got); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} - -func TestWithDescription(t *testing.T) { - type args struct { - desc string - } - type want struct { - want instrument.Option - } - type test struct { - name string - args args - want want - checkFunc func(want, instrument.Option) error - beforeFunc func(*testing.T, args) - afterFunc func(*testing.T, args) - } - defaultCheckFunc := func(w want, got instrument.Option) error { - if !reflect.DeepEqual(got, w.want) { - return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) - } - return nil - } - tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - desc:"", - }, - want: want{}, - checkFunc: defaultCheckFunc, - beforeFunc: func(t *testing.T, args args) { - t.Helper() - }, - afterFunc: func(t *testing.T, args args) { - t.Helper() - }, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - desc:"", - }, - want: want{}, - checkFunc: defaultCheckFunc, - beforeFunc: func(t *testing.T, args args) { - t.Helper() - }, - afterFunc: func(t *testing.T, args args) { - t.Helper() - }, - } - }(), - */ - } - - for _, tc := range tests { - test := tc - t.Run(test.name, func(tt *testing.T) { - tt.Parallel() - defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) - if test.beforeFunc != nil { - test.beforeFunc(tt, test.args) - } - if test.afterFunc != nil { - defer test.afterFunc(tt, test.args) - } - checkFunc := test.checkFunc - if test.checkFunc == nil { - checkFunc = defaultCheckFunc - } - - got := WithDescription(test.args.desc) - if err := checkFunc(test.want, got); err != nil { - tt.Errorf("error = %v", err) - } - }) - } -} diff --git a/internal/observability/metrics/runtime/cgo/cgo.go b/internal/observability/metrics/runtime/cgo/cgo.go index 3faf6be70cd..57b2d935f2f 100644 --- a/internal/observability/metrics/runtime/cgo/cgo.go +++ b/internal/observability/metrics/runtime/cgo/cgo.go @@ -18,8 +18,9 @@ import ( "runtime" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -33,22 +34,22 @@ func New() metrics.Metric { return &cgo{} } -func (*cgo) View() ([]*metrics.View, error) { - count, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &count, +func (*cgo) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } func (*cgo) Register(m metrics.Meter) error { - count, err := m.AsyncInt64().Gauge( + count, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -56,12 +57,11 @@ func (*cgo) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - count, - }, - func(ctx context.Context) { - count.Observe(ctx, int64(runtime.NumGoroutine())) - }, + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { + o.ObserveInt64(count, runtime.NumCgoCall()) + return nil + }, count, ) + return err } diff --git a/internal/observability/metrics/runtime/goroutine/goroutine.go b/internal/observability/metrics/runtime/goroutine/goroutine.go index b094f4feadc..5349d89ed9b 100644 --- a/internal/observability/metrics/runtime/goroutine/goroutine.go +++ b/internal/observability/metrics/runtime/goroutine/goroutine.go @@ -18,8 +18,9 @@ import ( "runtime" "github.com/vdaas/vald/internal/observability/metrics" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -33,23 +34,22 @@ func New() metrics.Metric { return &goroutine{} } -func (*goroutine) View() ([]*metrics.View, error) { - count, err := view.New( - view.MatchInstrumentName(metricsName), - view.WithSetDescription(metricsDescription), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - - return []*metrics.View{ - &count, +func (*goroutine) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: metricsName, + Description: metricsDescription, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } -func (*goroutine) Register(m metrics.Meter) error { - conter, err := m.AsyncInt64().Gauge( +func (*goroutine) Register(m metrics.Meter) (err error) { + counter, err := m.Int64ObservableGauge( metricsName, metrics.WithDescription(metricsDescription), metrics.WithUnit(metrics.Dimensionless), @@ -57,12 +57,11 @@ func (*goroutine) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - conter, - }, - func(ctx context.Context) { - conter.Observe(ctx, int64(runtime.NumGoroutine())) - }, + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { + o.ObserveInt64(counter, int64(runtime.NumGoroutine())) + return nil + }, counter, ) + return err } diff --git a/internal/observability/metrics/version/version.go b/internal/observability/metrics/version/version.go index 01eeced31f8..d1cda42eb4f 100644 --- a/internal/observability/metrics/version/version.go +++ b/internal/observability/metrics/version/version.go @@ -22,8 +22,9 @@ import ( "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/metrics" "github.com/vdaas/vald/internal/strings" + api "go.opentelemetry.io/otel/metric" + view "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/aggregation" - "go.opentelemetry.io/otel/sdk/metric/view" ) const ( @@ -76,22 +77,22 @@ func labelKVs(labels ...string) map[string]string { return info } -func (*version) View() ([]*metrics.View, error) { - otlv, err := view.New( - view.MatchInstrumentName(name), - view.WithSetDescription(description), - view.WithSetAggregation(aggregation.LastValue{}), - ) - if err != nil { - return nil, err - } - return []*metrics.View{ - &otlv, +func (*version) View() ([]metrics.View, error) { + return []metrics.View{ + view.NewView( + view.Instrument{ + Name: name, + Description: description, + }, + view.Stream{ + Aggregation: aggregation.LastValue{}, + }, + ), }, nil } -func (v *version) Register(m metrics.Meter) error { - info, err := m.AsyncInt64().Gauge( +func (v *version) Register(m metrics.Meter) (err error) { + info, err := m.Int64ObservableGauge( name, metrics.WithDescription(description), metrics.WithUnit(metrics.Dimensionless), @@ -99,16 +100,15 @@ func (v *version) Register(m metrics.Meter) error { if err != nil { return err } - return m.RegisterCallback( - []metrics.AsynchronousInstrument{ - info, - }, - func(ctx context.Context) { + _, err = m.RegisterCallback( + func(_ context.Context, o api.Observer) error { attrs := make([]attribute.KeyValue, 0, len(v.kvs)) for key, val := range v.kvs { attrs = append(attrs, attribute.String(key, val)) } - info.Observe(ctx, 1, attrs...) - }, + o.ObserveInt64(info, 1, attrs...) + return nil + }, info, ) + return err } diff --git a/internal/observability/trace/status.go b/internal/observability/trace/status.go index 199d4175a00..86f0d942515 100644 --- a/internal/observability/trace/status.go +++ b/internal/observability/trace/status.go @@ -21,7 +21,7 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "go.opentelemetry.io/otel/attribute" ocodes "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.12.0" + semconv "go.opentelemetry.io/otel/semconv/v1.20.0" ) type Attributes = []attribute.KeyValue diff --git a/k8s/agent/daemonset.yaml b/k8s/agent/faiss/configmap.yaml similarity index 100% rename from k8s/agent/daemonset.yaml rename to k8s/agent/faiss/configmap.yaml diff --git a/k8s/agent/configmap.yaml b/k8s/agent/ngt/configmap.yaml similarity index 98% rename from k8s/agent/configmap.yaml rename to k8s/agent/ngt/configmap.yaml index 2551e454139..d2fabd0f644 100644 --- a/k8s/agent/configmap.yaml +++ b/k8s/agent/ngt/configmap.yaml @@ -17,7 +17,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: vald-agent-ngt-config + name: vald-agent-config labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.6 @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -181,7 +182,7 @@ data: namespace: "_MY_POD_NAMESPACE_" pod_name: "_MY_POD_NAME_" node_name: "_MY_NODE_NAME_" - service_name: "vald-agent-ngt" + service_name: "vald-agent" metrics: enable_cgo: true enable_goroutine: true @@ -195,7 +196,7 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false ngt: diff --git a/k8s/agent/pdb.yaml b/k8s/agent/pdb.yaml index c58182ba489..2b9d5cf04d4 100644 --- a/k8s/agent/pdb.yaml +++ b/k8s/agent/pdb.yaml @@ -17,7 +17,7 @@ apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: vald-agent-ngt + name: vald-agent labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.6 @@ -29,4 +29,4 @@ spec: maxUnavailable: 1 selector: matchLabels: - app: vald-agent-ngt + app: vald-agent diff --git a/k8s/agent/priorityclass.yaml b/k8s/agent/priorityclass.yaml index 27c87adf2f5..e7e4440ea2c 100644 --- a/k8s/agent/priorityclass.yaml +++ b/k8s/agent/priorityclass.yaml @@ -17,7 +17,7 @@ apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: - name: default-vald-agent-ngt-priority + name: default-vald-agent-priority labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.6 diff --git a/k8s/agent/deployment.yaml b/k8s/agent/qbg/configmap.yaml similarity index 100% rename from k8s/agent/deployment.yaml rename to k8s/agent/qbg/configmap.yaml diff --git a/k8s/agent/sidecar-configmap.yaml b/k8s/agent/sidecar/configmap.yaml similarity index 100% rename from k8s/agent/sidecar-configmap.yaml rename to k8s/agent/sidecar/configmap.yaml diff --git a/k8s/agent/sidecar-svc.yaml b/k8s/agent/sidecar/svc.yaml similarity index 100% rename from k8s/agent/sidecar-svc.yaml rename to k8s/agent/sidecar/svc.yaml diff --git a/k8s/agent/statefulset.yaml b/k8s/agent/statefulset.yaml index 5184b6e0567..8bacea6ee37 100644 --- a/k8s/agent/statefulset.yaml +++ b/k8s/agent/statefulset.yaml @@ -17,9 +17,9 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: vald-agent-ngt + name: vald-agent labels: - app: vald-agent-ngt + app: vald-agent app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.6 app.kubernetes.io/managed-by: Helm @@ -27,13 +27,13 @@ metadata: app.kubernetes.io/version: v1.7.6 app.kubernetes.io/component: agent spec: - serviceName: vald-agent-ngt + serviceName: vald-agent podManagementPolicy: Parallel replicas: 5 revisionHistoryLimit: 2 selector: matchLabels: - app: vald-agent-ngt + app: vald-agent updateStrategy: type: RollingUpdate rollingUpdate: @@ -42,7 +42,7 @@ spec: metadata: creationTimestamp: null labels: - app: vald-agent-ngt + app: vald-agent app.kubernetes.io/name: vald app.kubernetes.io/instance: release-name app.kubernetes.io/component: agent @@ -51,7 +51,7 @@ spec: profefe.com/port: "6060" profefe.com/service: vald-agent-ngt pyroscope.io/scrape: "true" - pyroscope.io/application-name: vald-agent-ngt + pyroscope.io/application-name: vald-agent pyroscope.io/profile-cpu-enabled: "true" pyroscope.io/profile-mem-enabled: "true" pyroscope.io/port: "6060" @@ -70,12 +70,12 @@ spec: - key: app operator: In values: - - vald-agent-ngt + - vald-agent topologyKey: kubernetes.io/hostname weight: 100 requiredDuringSchedulingIgnoredDuringExecution: [] containers: - - name: vald-agent-ngt + - name: vald-agent image: "vdaas/vald-agent-ngt:nightly" imagePullPolicy: Always livenessProbe: @@ -151,7 +151,7 @@ spec: fieldRef: fieldPath: metadata.namespace volumeMounts: - - name: vald-agent-ngt-config + - name: vald-agent-config mountPath: /etc/server/ dnsPolicy: ClusterFirst restartPolicy: Always @@ -164,9 +164,9 @@ spec: runAsUser: 65532 terminationGracePeriodSeconds: 120 volumes: - - name: vald-agent-ngt-config + - name: vald-agent-config configMap: defaultMode: 420 - name: vald-agent-ngt-config - priorityClassName: default-vald-agent-ngt-priority + name: vald-agent-config + priorityClassName: default-vald-agent-priority status: diff --git a/k8s/agent/svc.yaml b/k8s/agent/svc.yaml index 28e1da4deea..05205a38324 100644 --- a/k8s/agent/svc.yaml +++ b/k8s/agent/svc.yaml @@ -17,7 +17,7 @@ apiVersion: v1 kind: Service metadata: - name: vald-agent-ngt + name: vald-agent labels: app.kubernetes.io/name: vald helm.sh/chart: vald-v1.7.6 diff --git a/k8s/discoverer/configmap.yaml b/k8s/discoverer/configmap.yaml index bc887bbd309..30843209525 100644 --- a/k8s/discoverer/configmap.yaml +++ b/k8s/discoverer/configmap.yaml @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -195,7 +196,7 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false discoverer: @@ -215,7 +216,7 @@ data: pod_metrics: labels: {} fields: - containers.name: vald-agent-ngt + containers.name: vald-agent node_metrics: labels: {} fields: {} diff --git a/k8s/discoverer/deployment.yaml b/k8s/discoverer/deployment.yaml index c83cb70f4a2..658aa91967e 100644 --- a/k8s/discoverer/deployment.yaml +++ b/k8s/discoverer/deployment.yaml @@ -47,7 +47,7 @@ spec: app.kubernetes.io/instance: release-name app.kubernetes.io/component: discoverer annotations: - checksum/configmap: 93b7c3aa6b8f0c29c84cb8f89df27119858e4174f0956120fdc63b944ec85d11 + checksum/configmap: 3e4acf4a9ab0eb280c2aaf4e40223ec78c2c8c7d43dde95bfc6bc9a544124e79 profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: vald-discoverer diff --git a/k8s/gateway/lb/configmap.yaml b/k8s/gateway/lb/configmap.yaml index 18bd5f5f24c..7a20b3c6b06 100644 --- a/k8s/gateway/lb/configmap.yaml +++ b/k8s/gateway/lb/configmap.yaml @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -195,13 +196,13 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false gateway: agent_port: 8081 - agent_name: "vald-agent-ngt" - agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_name: "vald-agent" + agent_dns: vald-agent.default.svc.cluster.local agent_namespace: "_MY_POD_NAMESPACE_" node_name: "" index_replica: 3 diff --git a/k8s/gateway/lb/deployment.yaml b/k8s/gateway/lb/deployment.yaml index c588528365e..09116547c46 100644 --- a/k8s/gateway/lb/deployment.yaml +++ b/k8s/gateway/lb/deployment.yaml @@ -46,7 +46,7 @@ spec: app.kubernetes.io/instance: release-name app.kubernetes.io/component: gateway-lb annotations: - checksum/configmap: f44923f86ffccf4b017a05493f282f179d663e4f51dde8807bbd82a98bcb298a + checksum/configmap: 97f13071366f89ae131e083c24284803575e1819f77c79fa025eb9c924d963f5 profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: vald-lb-gateway @@ -75,7 +75,7 @@ spec: - -e - -c - | - until [ "$(wget --server-response --spider --quiet http://vald-agent-ngt.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + until [ "$(wget --server-response --spider --quiet http://vald-agent.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do echo "waiting for agent to be ready..." sleep 2; done diff --git a/k8s/gateway/lb/ing.yaml b/k8s/gateway/lb/ing.yaml index ed97d539c09..8e2bd32eca0 100644 --- a/k8s/gateway/lb/ing.yaml +++ b/k8s/gateway/lb/ing.yaml @@ -1 +1,47 @@ --- +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/grpc-backend: "true" + labels: + name: vald-lb-gateway-ingress + app: vald-lb-gateway-ingress + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.6 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.6 + app.kubernetes.io/component: gateway-lb + name: vald-lb-gateway-ingress +spec: + defaultBackend: + service: + name: vald-lb-gateway + port: + name: grpc + rules: + - host: lb.gateway.vald.vdaas.org + http: + paths: + - backend: + service: + name: vald-lb-gateway + port: + name: grpc + pathType: ImplementationSpecific diff --git a/k8s/manager/index/configmap.yaml b/k8s/manager/index/configmap.yaml index 368338c4e58..1eaf78e705a 100644 --- a/k8s/manager/index/configmap.yaml +++ b/k8s/manager/index/configmap.yaml @@ -42,6 +42,7 @@ data: grpc: bidirectional_stream_concurrency: 20 connection_timeout: "" + enable_admin: true enable_reflection: true header_table_size: 0 initial_conn_window_size: 2097152 @@ -195,13 +196,13 @@ data: - go_version - go_os - go_arch - - ngt_version + - algorithm_info trace: enabled: false indexer: agent_port: 8081 - agent_name: "vald-agent-ngt" - agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_name: "vald-agent" + agent_dns: vald-agent.default.svc.cluster.local agent_namespace: "_MY_POD_NAMESPACE_" node_name: "" discoverer: diff --git a/k8s/manager/index/deployment.yaml b/k8s/manager/index/deployment.yaml index aa540909d88..533f8b171d7 100644 --- a/k8s/manager/index/deployment.yaml +++ b/k8s/manager/index/deployment.yaml @@ -47,7 +47,7 @@ spec: app.kubernetes.io/instance: release-name app.kubernetes.io/component: manager-index annotations: - checksum/configmap: 48604cf92536492933ac63941bb1e6cff3f72bf461b97742a524788363a0c13e + checksum/configmap: dd03ccb8811d4147fd533a3e56978b40d36f08ab7f9f3daa122f1668058f14e7 profefe.com/enable: "true" profefe.com/port: "6060" profefe.com/service: vald-manager-index @@ -65,7 +65,7 @@ spec: - -e - -c - | - until [ "$(wget --server-response --spider --quiet http://vald-agent-ngt.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + until [ "$(wget --server-response --spider --quiet http://vald-agent.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do echo "waiting for agent to be ready..." sleep 2; done diff --git a/k8s/metrics/grafana/dashboards/01-vald-agent.yaml b/k8s/metrics/grafana/dashboards/01-vald-agent.yaml index 3e10ce103ee..fcfb71a34bc 100644 --- a/k8s/metrics/grafana/dashboards/01-vald-agent.yaml +++ b/k8s/metrics/grafana/dashboards/01-vald-agent.yaml @@ -1066,7 +1066,7 @@ data: "calcs": [ "lastNotNull" ], - "fields": "/^ngt_version$/", + "fields": "/^algorithm_info$/", "values": false }, "text": {}, @@ -1080,7 +1080,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "label_replace(app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}, \"ngt_version\", \"v$1\", \"ngt_version\", \"([^v].*)\")", + "expr": "label_replace(app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}, \"algorithm_info\", \"v$1\", \"algorithm_info\", \"([^v].*)\")", "format": "table", "instant": true, "interval": "", diff --git a/k8s/operator/helm/crds/valdrelease.yaml b/k8s/operator/helm/crds/valdrelease.yaml index 9f135ab791b..4106e79335d 100644 --- a/k8s/operator/helm/crds/valdrelease.yaml +++ b/k8s/operator/helm/crds/valdrelease.yaml @@ -116,6 +116,12 @@ spec: items: type: object x-kubernetes-preserve-unknown-fields: true + algorithm: + type: string + enum: + - ngt + - qbg + - faiss annotations: type: object x-kubernetes-preserve-unknown-fields: true @@ -314,7 +320,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -732,6 +738,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -1222,7 +1230,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -1592,6 +1600,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2068,7 +2078,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -2429,6 +2439,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -2879,7 +2891,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -3277,6 +3289,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -4240,7 +4254,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -4638,6 +4652,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -5399,7 +5415,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -5797,6 +5813,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: @@ -6536,7 +6554,7 @@ spec: - go_os - go_arch - cgo_enabled - - ngt_version + - algorithm_info - build_cpu_info_flags otlp: type: object @@ -6937,6 +6955,8 @@ spec: type: integer connection_timeout: type: string + enable_admin: + type: boolean enable_reflection: type: boolean header_table_size: diff --git a/pkg/agent/core/faiss/config/config.go b/pkg/agent/core/faiss/config/config.go new file mode 100644 index 00000000000..8952abd7157 --- /dev/null +++ b/pkg/agent/core/faiss/config/config.go @@ -0,0 +1,77 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package setting stores all server application settings +package config + +import ( + "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" +) + +// GlobalConfig is type alias for config.GlobalConfig. +type GlobalConfig = config.GlobalConfig + +// Data represent a application setting data content (config.yaml). +// In K8s environment, this configuration is stored in K8s ConfigMap. +type Data struct { + GlobalConfig `json:",inline" yaml:",inline"` + + // Server represent all server configurations + Server *config.Servers `json:"server_config" yaml:"server_config"` + + // Observability represent observability configurations + Observability *config.Observability `json:"observability" yaml:"observability"` + + // Faiss represent faiss core configuration + Faiss *config.Faiss `json:"faiss" yaml:"faiss"` +} + +// NewConfig returns the Data struct or error from the given file path. +func NewConfig(path string) (cfg *Data, err error) { + cfg = new(Data) + + err = config.Read(path, &cfg) + if err != nil { + return nil, err + } + + if cfg != nil { + cfg.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Server != nil { + cfg.Server = cfg.Server.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Observability != nil { + cfg.Observability = cfg.Observability.Bind() + } else { + cfg.Observability = new(config.Observability).Bind() + } + + if cfg.Faiss != nil { + cfg.Faiss = cfg.Faiss.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + return cfg, nil +} diff --git a/pkg/agent/core/faiss/handler/doc.go b/pkg/agent/core/faiss/handler/doc.go new file mode 100644 index 00000000000..825bcc7f61f --- /dev/null +++ b/pkg/agent/core/faiss/handler/doc.go @@ -0,0 +1,17 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package handler diff --git a/pkg/agent/core/faiss/handler/grpc/handler.go b/pkg/agent/core/faiss/handler/grpc/handler.go new file mode 100644 index 00000000000..89debe0b090 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/handler.go @@ -0,0 +1,95 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package grpc provides grpc server logic +package grpc + +import ( + "reflect" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" +) + +type Server interface { + agent.AgentServer + vald.Server +} + +type server struct { + name string + ip string + faiss service.Faiss + eg errgroup.Group + streamConcurrency int + agent.UnimplementedAgentServer + vald.UnimplementedValdServer +} + +const ( + apiName = "vald/agent/core/faiss" + faissResourceType = "vald/internal/core/algorithm" +) + +var errFaiss = new(errors.FaissError) + +func New(opts ...Option) (Server, error) { + s := new(server) + + for _, opt := range append(defaultOptions, opts...) { + if err := opt(s); err != nil { + werr := errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + + e := new(errors.ErrCriticalOption) + if errors.As(err, &e) { + log.Error(werr) + return nil, werr + } + log.Warn(werr) + } + } + return s, nil +} + +func (s *server) newLocations(uuids ...string) (locs *payload.Object_Locations) { + if len(uuids) == 0 { + return nil + } + locs = &payload.Object_Locations{ + Locations: make([]*payload.Object_Location, 0, len(uuids)), + } + for _, uuid := range uuids { + locs.Locations = append(locs.GetLocations(), &payload.Object_Location{ + Name: s.name, + Uuid: uuid, + Ips: []string{s.ip}, + }) + } + return locs +} + +func (s *server) newLocation(uuid string) *payload.Object_Location { + locs := s.newLocations(uuid) + if locs != nil && locs.GetLocations() != nil && len(locs.GetLocations()) > 0 { + return locs.Locations[0] + } + return nil +} diff --git a/pkg/agent/core/faiss/handler/grpc/index.go b/pkg/agent/core/faiss/handler/grpc/index.go new file mode 100644 index 00000000000..0ea6995e8ca --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/index.go @@ -0,0 +1,180 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" +) + +func (s *server) CreateIndex(ctx context.Context, c *payload.Control_CreateIndexRequest) (res *payload.Empty, err error) { + ctx, span := trace.StartSpan(ctx, apiName+".CreateIndex") + defer func() { + if span != nil { + span.End() + } + }() + res = new(payload.Empty) + err = s.faiss.CreateIndex(ctx) + if err != nil { + if errors.Is(err, errors.ErrUncommittedIndexNotFound) { + err = status.WrapWithFailedPrecondition(fmt.Sprintf("CreateIndex API failed"), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, + &errdetails.PreconditionFailure{ + Violations: []*errdetails.PreconditionFailureViolation{ + { + Type: "uncommitted index is empty", + Subject: "failed to CreateIndex operation caused by empty uncommitted indices", + }, + }, + }, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeFailedPrecondition(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + log.Error(err) + err = status.WrapWithInternal(fmt.Sprintf("CreateIndex API failed"), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInternal(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return res, nil +} + +func (s *server) SaveIndex(ctx context.Context, _ *payload.Empty) (res *payload.Empty, err error) { + ctx, span := trace.StartSpan(ctx, apiName+".SaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + res = new(payload.Empty) + err = s.faiss.SaveIndex(ctx) + if err != nil { + log.Error(err) + err = status.WrapWithInternal("SaveIndex API failed to save indices", err, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.SaveIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInternal(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return res, nil +} + +func (s *server) CreateAndSaveIndex(ctx context.Context, c *payload.Control_CreateIndexRequest) (res *payload.Empty, err error) { + ctx, span := trace.StartSpan(ctx, apiName+".CreateAndSaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + res = new(payload.Empty) + err = s.faiss.CreateAndSaveIndex(ctx) + if err != nil { + if errors.Is(err, errors.ErrUncommittedIndexNotFound) { + err = status.WrapWithFailedPrecondition(fmt.Sprintf("CreateAndSaveIndex API failed to create indexes pool_size = %d", c.GetPoolSize()), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateAndSaveIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, + &errdetails.PreconditionFailure{ + Violations: []*errdetails.PreconditionFailureViolation{ + { + Type: "uncommitted index is empty", + Subject: "failed to CreateAndSaveIndex operation caused by empty uncommitted indices", + }, + }, + }, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeFailedPrecondition(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + err = status.WrapWithInternal(fmt.Sprintf("CreateAndSaveIndex API failed to create indexes pool_size = %d", c.GetPoolSize()), err, + &errdetails.RequestInfo{ + ServingData: errdetails.Serialize(c), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.CreateAndSaveIndex", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInternal(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return res, nil +} + +func (s *server) IndexInfo(ctx context.Context, c *payload.Empty) (res *payload.Info_Index_Count, err error) { + _, span := trace.StartSpan(ctx, apiName+".IndexInfo") + defer func() { + if span != nil { + span.End() + } + }() + + return &payload.Info_Index_Count{ + Stored: uint32(s.faiss.Len()), + Uncommitted: uint32(s.faiss.InsertVQueueBufferLen() + s.faiss.DeleteVQueueBufferLen()), + Indexing: s.faiss.IsIndexing(), + Saving: s.faiss.IsSaving(), + }, nil +} diff --git a/pkg/agent/core/faiss/handler/grpc/insert.go b/pkg/agent/core/faiss/handler/grpc/insert.go new file mode 100644 index 00000000000..44087a9deac --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/insert.go @@ -0,0 +1,141 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Insert(ctx context.Context, req *payload.Insert_Request) (res *payload.Object_Location, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.InsertRPCName) + defer func() { + if span != nil { + span.End() + } + }() + vec := req.GetVector() + if len(vec.GetVector()) != s.faiss.GetDimensionSize() { + err = errors.ErrIncompatibleDimensionSize(len(vec.GetVector()), int(s.faiss.GetDimensionSize())) + err = status.WrapWithInvalidArgument("Insert API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: vec.GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + err = s.faiss.InsertWithTime(vec.GetId(), vec.GetVector(), req.GetConfig().GetTimestamp()) + if err != nil { + var attrs []attribute.KeyValue + + if errors.Is(err, errors.ErrUUIDAlreadyExists(vec.GetId())) { + err = status.WrapWithAlreadyExists(fmt.Sprintf("Insert API uuid %s already exists", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeAlreadyExists(err.Error()) + } else if errors.Is(err, errors.ErrUUIDNotFound(0)) { + err = status.WrapWithInvalidArgument(fmt.Sprintf("Insert API empty uuid \"%s\" was given", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + } else { + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Internal, + "failed to parse Insert gRPC error response", + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Insert", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + attrs = trace.FromGRPCStatus(st.Code(), msg) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return s.newLocation(vec.GetId()), nil +} + +func (s *server) StreamInsert(stream vald.Insert_StreamInsertServer) (err error) { + return s.UnimplementedValdServer.UnimplementedInsertServer.StreamInsert(stream) +} + +func (s *server) MultiInsert(ctx context.Context, reqs *payload.Insert_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedInsertServer.MultiInsert(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/linear_search.go b/pkg/agent/core/faiss/handler/grpc/linear_search.go new file mode 100644 index 00000000000..3fb17d836ec --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/linear_search.go @@ -0,0 +1,48 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" +) + +func (s *server) LinearSearch(ctx context.Context, req *payload.Search_Request) (res *payload.Search_Response, err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.LinearSearch(ctx, req) +} + +func (s *server) LinearSearchByID(ctx context.Context, req *payload.Search_IDRequest) (res *payload.Search_Response, err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.LinearSearchByID(ctx, req) +} + +func (s *server) StreamLinearSearch(stream vald.Search_StreamLinearSearchServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamLinearSearch(stream) +} + +func (s *server) StreamLinearSearchByID(stream vald.Search_StreamLinearSearchByIDServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamLinearSearchByID(stream) +} + +func (s *server) MultiLinearSearch(ctx context.Context, reqs *payload.Search_MultiRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiLinearSearch(ctx, reqs) +} + +func (s *server) MultiLinearSearchByID(ctx context.Context, reqs *payload.Search_MultiIDRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiLinearSearchByID(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/object.go b/pkg/agent/core/faiss/handler/grpc/object.go new file mode 100644 index 00000000000..ce3e4bbdfaf --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/object.go @@ -0,0 +1,96 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" +) + +func (s *server) Exists(ctx context.Context, uid *payload.Object_ID) (res *payload.Object_ID, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.ExistsRPCName) + defer func() { + if span != nil { + span.End() + } + }() + uuid := uid.GetId() + if len(uuid) == 0 { + err = errors.ErrInvalidUUID(uuid) + err = status.WrapWithInvalidArgument(fmt.Sprintf("Exists API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(uid), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Exists", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + log.Warn(err) + return nil, err + } + if _, ok := s.faiss.Exists(uuid); !ok { + err = errors.ErrObjectIDNotFound(uid.GetId()) + err = status.WrapWithNotFound(fmt.Sprintf("Exists API meta %s's uuid not found", uid.GetId()), err, + &errdetails.RequestInfo{ + RequestId: uid.GetId(), + ServingData: errdetails.Serialize(uid), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Exists", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, + uid.GetId()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeNotFound(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return uid, nil +} + +func (s *server) GetObject(ctx context.Context, id *payload.Object_VectorRequest) (res *payload.Object_Vector, err error) { + return s.UnimplementedValdServer.UnimplementedObjectServer.GetObject(ctx, id) +} + +func (s *server) StreamGetObject(stream vald.Object_StreamGetObjectServer) (err error) { + return s.UnimplementedValdServer.UnimplementedObjectServer.StreamGetObject(stream) +} diff --git a/pkg/agent/core/faiss/handler/grpc/option.go b/pkg/agent/core/faiss/handler/grpc/option.go new file mode 100644 index 00000000000..2b443b1cfe4 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/option.go @@ -0,0 +1,100 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package grpc provides grpc server logic +package grpc + +import ( + "os" + "runtime" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" +) + +// Option represents the functional option for server. +type Option func(*server) error + +var defaultOptions = []Option{ + WithName(func() string { + name, err := os.Hostname() + if err != nil { + log.Warn(err) + } + return name + }()), + WithIP(net.LoadLocalIP()), + WithStreamConcurrency(runtime.GOMAXPROCS(-1) * 10), + WithErrGroup(errgroup.Get()), +} + +// WithIP returns the option to set the IP for server. +func WithIP(ip string) Option { + return func(s *server) error { + if len(ip) == 0 { + return errors.NewErrInvalidOption("ip", ip) + } + s.ip = ip + return nil + } +} + +// WithName returns the option to set the name for server. +func WithName(name string) Option { + return func(s *server) error { + if len(name) == 0 { + return errors.NewErrInvalidOption("name", name) + } + s.name = name + return nil + } +} + +// WithFaiss returns the option to set the Faiss service for server. +func WithFaiss(f service.Faiss) Option { + return func(s *server) error { + if f == nil { + return errors.NewErrInvalidOption("faiss", f) + } + s.faiss = f + return nil + } +} + +// WithStreamConcurrency returns the option to set the stream concurrency for server. +func WithStreamConcurrency(c int) Option { + return func(s *server) error { + if c <= 0 { + return errors.NewErrInvalidOption("streamConcurrency", c) + } + s.streamConcurrency = c + return nil + } +} + +// WithErrGroup returns the option to set the error group for server. +func WithErrGroup(eg errgroup.Group) Option { + return func(s *server) error { + if eg == nil { + return errors.NewErrInvalidOption("errGroup", eg) + } + s.eg = eg + return nil + } +} diff --git a/pkg/agent/core/faiss/handler/grpc/remove.go b/pkg/agent/core/faiss/handler/grpc/remove.go new file mode 100644 index 00000000000..96b858de914 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/remove.go @@ -0,0 +1,134 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Remove(ctx context.Context, req *payload.Remove_Request) (res *payload.Object_Location, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.RemoveRPCName) + defer func() { + if span != nil { + span.End() + } + }() + id := req.GetId() + uuid := id.GetId() + if len(uuid) == 0 { + err = errors.ErrInvalidUUID(uuid) + err = status.WrapWithInvalidArgument(fmt.Sprintf("Remove API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + log.Warn(err) + return nil, err + } + err = s.faiss.DeleteWithTime(uuid, req.GetConfig().GetTimestamp()) + if err != nil { + var attrs []attribute.KeyValue + if errors.Is(err, errors.ErrObjectIDNotFound(uuid)) { + err = status.WrapWithNotFound(fmt.Sprintf("Remove API uuid %s not found", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeNotFound(err.Error()) + } else if errors.Is(err, errors.ErrUUIDNotFound(0)) { + err = status.WrapWithInvalidArgument(fmt.Sprintf("Remove API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + } else { + err = status.WrapWithInternal("Remove API failed", err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Remove", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + return s.newLocation(uuid), nil +} + +func (s *server) StreamRemove(stream vald.Remove_StreamRemoveServer) (err error) { + return s.UnimplementedValdServer.UnimplementedRemoveServer.StreamRemove(stream) +} + +func (s *server) MultiRemove(ctx context.Context, reqs *payload.Remove_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedRemoveServer.MultiRemove(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/search.go b/pkg/agent/core/faiss/handler/grpc/search.go new file mode 100644 index 00000000000..8aecfb6e7a8 --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/search.go @@ -0,0 +1,196 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/pkg/agent/core/faiss/model" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Search(ctx context.Context, req *payload.Search_Request) (res *payload.Search_Response, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.SearchRPCName) + defer func() { + if span != nil { + span.End() + } + }() + if len(req.GetVector()) != s.faiss.GetDimensionSize() { + err = errors.ErrIncompatibleDimensionSize(len(req.GetVector()), int(s.faiss.GetDimensionSize())) + err = status.WrapWithInvalidArgument("Search API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + }) + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + res, err = toSearchResponse( + s.faiss.Search( + req.GetConfig().GetNum(), + 1, + req.GetVector())) + if err != nil || res == nil { + var attrs []attribute.KeyValue + switch { + case errors.Is(err, errors.ErrCreateIndexingIsInProgress): + err = status.WrapWithAborted("Search API aborted to process search request due to createing indices is in progress", err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Debug(err) + attrs = trace.StatusCodeAborted(err.Error()) + case errors.Is(err, errors.ErrEmptySearchResult), + err == nil && res == nil, + 0 < req.GetConfig().GetMinNum() && len(res.GetResults()) < int(req.GetConfig().GetMinNum()): + err = status.WrapWithNotFound(fmt.Sprintf("Search API requestID %s's search result not found", req.GetConfig().GetRequestId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Debug(err) + attrs = trace.StatusCodeNotFound(err.Error()) + case errors.As(err, &errFaiss): + log.Errorf("faiss core process returned error: %v", err) + err = status.WrapWithInternal("Search API failed to process search request due to faiss core process returned error", err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search/core.faiss", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + case errors.Is(err, errors.ErrIncompatibleDimensionSize(len(req.GetVector()), int(s.faiss.GetDimensionSize()))): + err = status.WrapWithInvalidArgument("Search API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + default: + err = status.WrapWithInternal("Search API failed to process search request", err, + &errdetails.RequestInfo{ + RequestId: req.GetConfig().GetRequestId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Search", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + res.RequestId = req.GetConfig().GetRequestId() + return res, nil +} + +func (s *server) SearchByID(ctx context.Context, req *payload.Search_IDRequest) (res *payload.Search_Response, err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.SearchByID(ctx, req) +} + +func (s *server) StreamSearch(stream vald.Search_StreamSearchServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamSearch(stream) +} + +func (s *server) StreamSearchByID(stream vald.Search_StreamSearchByIDServer) (err error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.StreamSearchByID(stream) +} + +func (s *server) MultiSearch(ctx context.Context, reqs *payload.Search_MultiRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiSearch(ctx, reqs) +} + +func (s *server) MultiSearchByID(ctx context.Context, reqs *payload.Search_MultiIDRequest) (res *payload.Search_Responses, errs error) { + return s.UnimplementedValdServer.UnimplementedSearchServer.MultiSearchByID(ctx, reqs) +} + +func toSearchResponse(dists []model.Distance, err error) (res *payload.Search_Response, rerr error) { + if err != nil { + return nil, err + } + if len(dists) == 0 { + return nil, errors.ErrEmptySearchResult + } + res = new(payload.Search_Response) + res.Results = make([]*payload.Object_Distance, 0, len(dists)) + for _, dist := range dists { + res.Results = append(res.GetResults(), &payload.Object_Distance{ + Id: dist.ID, + Distance: dist.Distance, + }) + } + return res, nil +} diff --git a/pkg/agent/core/faiss/handler/grpc/update.go b/pkg/agent/core/faiss/handler/grpc/update.go new file mode 100644 index 00000000000..f8762eaea6e --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/update.go @@ -0,0 +1,173 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + "fmt" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc/errdetails" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "go.opentelemetry.io/otel/attribute" +) + +func (s *server) Update(ctx context.Context, req *payload.Update_Request) (res *payload.Object_Location, err error) { + _, span := trace.StartSpan(ctx, apiName+"/"+vald.UpdateRPCName) + defer func() { + if span != nil { + span.End() + } + }() + + vec := req.GetVector() + if len(vec.GetVector()) != s.faiss.GetDimensionSize() { + err = errors.ErrIncompatibleDimensionSize(len(vec.GetVector()), int(s.faiss.GetDimensionSize())) + err = status.WrapWithInvalidArgument("Update API Incompatible Dimension Size detected", + err, + &errdetails.RequestInfo{ + RequestId: vec.GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "vector dimension size", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + uuid := vec.GetId() + if len(uuid) == 0 { + err = errors.ErrInvalidUUID(uuid) + err = status.WrapWithInvalidArgument(fmt.Sprintf("Update API invalid argument for uuid \"%s\" detected", uuid), err, + &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + return nil, err + } + + err = s.faiss.UpdateWithTime(uuid, vec.GetVector(), req.GetConfig().GetTimestamp()) + if err != nil { + var attrs []attribute.KeyValue + if errors.Is(err, errors.ErrObjectIDNotFound(vec.GetId())) { + err = status.WrapWithNotFound(fmt.Sprintf("Update API uuid %s not found", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeNotFound(err.Error()) + } else if errors.Is(err, errors.ErrUUIDNotFound(0)) || errors.Is(err, errors.ErrInvalidDimensionSize(len(vec.GetVector()), s.faiss.GetDimensionSize())) { + err = status.WrapWithInvalidArgument(fmt.Sprintf("Update API invalid argument for uuid \"%s\" vec \"%v\" detected", vec.GetId(), vec.GetVector()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "uuid or vector", + Description: err.Error(), + }, + }, + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeInvalidArgument(err.Error()) + } else if errors.Is(err, errors.ErrUUIDAlreadyExists(vec.GetId())) { + err = status.WrapWithAlreadyExists(fmt.Sprintf("Update API uuid %s's same data already exists", vec.GetId()), err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }) + log.Warn(err) + attrs = trace.StatusCodeAlreadyExists(err.Error()) + } else { + err = status.WrapWithInternal("Update API failed", err, + &errdetails.RequestInfo{ + RequestId: req.GetVector().GetId(), + ServingData: errdetails.Serialize(req), + }, + &errdetails.ResourceInfo{ + ResourceType: faissResourceType + "/faiss.Update", + ResourceName: fmt.Sprintf("%s: %s(%s)", apiName, s.name, s.ip), + }, info.Get()) + log.Error(err) + attrs = trace.StatusCodeInternal(err.Error()) + } + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + return s.newLocation(vec.GetId()), nil +} + +func (s *server) StreamUpdate(stream vald.Update_StreamUpdateServer) (err error) { + return s.UnimplementedValdServer.UnimplementedUpdateServer.StreamUpdate(stream) +} + +func (s *server) MultiUpdate(ctx context.Context, reqs *payload.Update_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedUpdateServer.MultiUpdate(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/grpc/upsert.go b/pkg/agent/core/faiss/handler/grpc/upsert.go new file mode 100644 index 00000000000..48227c4e98e --- /dev/null +++ b/pkg/agent/core/faiss/handler/grpc/upsert.go @@ -0,0 +1,36 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package grpc + +import ( + "context" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" +) + +func (s *server) Upsert(ctx context.Context, req *payload.Upsert_Request) (loc *payload.Object_Location, err error) { + return s.UnimplementedValdServer.UnimplementedUpsertServer.Upsert(ctx, req) +} + +func (s *server) StreamUpsert(stream vald.Upsert_StreamUpsertServer) (err error) { + return s.UnimplementedValdServer.UnimplementedUpsertServer.StreamUpsert(stream) +} + +func (s *server) MultiUpsert(ctx context.Context, reqs *payload.Upsert_MultiRequest) (res *payload.Object_Locations, err error) { + return s.UnimplementedValdServer.UnimplementedUpsertServer.MultiUpsert(ctx, reqs) +} diff --git a/pkg/agent/core/faiss/handler/rest/handler.go b/pkg/agent/core/faiss/handler/rest/handler.go new file mode 100644 index 00000000000..89e4c20d560 --- /dev/null +++ b/pkg/agent/core/faiss/handler/rest/handler.go @@ -0,0 +1,175 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package rest provides rest api logic +package rest + +import ( + "net/http" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/net/http/dump" + "github.com/vdaas/vald/internal/net/http/json" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/grpc" +) + +type Handler interface { + Index(w http.ResponseWriter, r *http.Request) (int, error) + Exists(w http.ResponseWriter, r *http.Request) (int, error) + Search(w http.ResponseWriter, r *http.Request) (int, error) + SearchByID(w http.ResponseWriter, r *http.Request) (int, error) + LinearSearch(w http.ResponseWriter, r *http.Request) (int, error) + LinearSearchByID(w http.ResponseWriter, r *http.Request) (int, error) + Insert(w http.ResponseWriter, r *http.Request) (int, error) + MultiInsert(w http.ResponseWriter, r *http.Request) (int, error) + Update(w http.ResponseWriter, r *http.Request) (int, error) + MultiUpdate(w http.ResponseWriter, r *http.Request) (int, error) + Remove(w http.ResponseWriter, r *http.Request) (int, error) + MultiRemove(w http.ResponseWriter, r *http.Request) (int, error) + CreateIndex(w http.ResponseWriter, r *http.Request) (int, error) + SaveIndex(w http.ResponseWriter, r *http.Request) (int, error) + CreateAndSaveIndex(w http.ResponseWriter, r *http.Request) (int, error) + GetObject(w http.ResponseWriter, r *http.Request) (int, error) +} + +type handler struct { + agent grpc.Server +} + +func New(opts ...Option) Handler { + h := new(handler) + + for _, opt := range append(defaultOptions, opts...) { + opt(h) + } + return h +} + +func (h *handler) Index(w http.ResponseWriter, r *http.Request) (int, error) { + data := make(map[string]interface{}) + return json.Handler(w, r, &data, func() (interface{}, error) { + return dump.Request(nil, data, r) + }) +} + +func (h *handler) Search(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Search(r.Context(), req) + }) +} + +func (h *handler) SearchByID(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_IDRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.SearchByID(r.Context(), req) + }) +} + +func (h *handler) LinearSearch(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.LinearSearch(r.Context(), req) + }) +} + +func (h *handler) LinearSearchByID(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Search_IDRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.LinearSearchByID(r.Context(), req) + }) +} + +func (h *handler) Insert(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Insert_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Insert(r.Context(), req) + }) +} + +func (h *handler) MultiInsert(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Insert_MultiRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.MultiInsert(r.Context(), req) + }) +} + +func (h *handler) Update(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Update_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Update(r.Context(), req) + }) +} + +func (h *handler) MultiUpdate(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Update_MultiRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.MultiUpdate(r.Context(), req) + }) +} + +func (h *handler) Remove(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Remove_Request + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Remove(r.Context(), req) + }) +} + +func (h *handler) MultiRemove(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Remove_MultiRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.MultiRemove(r.Context(), req) + }) +} + +func (h *handler) CreateIndex(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Control_CreateIndexRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.CreateIndex(r.Context(), req) + }) +} + +func (h *handler) SaveIndex(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Empty + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.SaveIndex(r.Context(), req) + }) +} + +func (h *handler) CreateAndSaveIndex(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Control_CreateIndexRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + _, err = h.agent.CreateIndex(r.Context(), req) + if err != nil { + return nil, err + } + return h.agent.SaveIndex(r.Context(), nil) + }) +} + +func (h *handler) GetObject(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Object_VectorRequest + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.GetObject(r.Context(), req) + }) +} + +func (h *handler) Exists(w http.ResponseWriter, r *http.Request) (code int, err error) { + var req *payload.Object_ID + return json.Handler(w, r, &req, func() (interface{}, error) { + return h.agent.Exists(r.Context(), req) + }) +} diff --git a/pkg/agent/core/faiss/handler/rest/option.go b/pkg/agent/core/faiss/handler/rest/option.go new file mode 100644 index 00000000000..9b3b41e7d64 --- /dev/null +++ b/pkg/agent/core/faiss/handler/rest/option.go @@ -0,0 +1,30 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package rest provides rest api logic +package rest + +import "github.com/vdaas/vald/pkg/agent/core/faiss/handler/grpc" + +type Option func(*handler) + +var defaultOptions = []Option{} + +func WithAgent(a grpc.Server) Option { + return func(h *handler) { + h.agent = a + } +} diff --git a/pkg/agent/core/faiss/model/faiss.go b/pkg/agent/core/faiss/model/faiss.go new file mode 100644 index 00000000000..5d751a83e38 --- /dev/null +++ b/pkg/agent/core/faiss/model/faiss.go @@ -0,0 +1,23 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package model defines object structure +package model + +type Distance struct { + ID string + Distance float32 +} diff --git a/pkg/agent/core/faiss/router/option.go b/pkg/agent/core/faiss/router/option.go new file mode 100644 index 00000000000..4db27948195 --- /dev/null +++ b/pkg/agent/core/faiss/router/option.go @@ -0,0 +1,51 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package router provides implementation of Go API for routing http Handler wrapped by rest.Func +package router + +import ( + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/rest" +) + +// Option represents the functional option for router. +type Option func(*router) + +var defaultOptions = []Option{ + WithTimeout("3s"), +} + +// WithHandler returns the option to set the handler for the router. +func WithHandler(h rest.Handler) Option { + return func(r *router) { + r.handler = h + } +} + +// WithTimeout returns the option to set the timeout for the router. +func WithTimeout(timeout string) Option { + return func(r *router) { + r.timeout = timeout + } +} + +// WithErrGroup returns the option to set the error group for the router. +func WithErrGroup(eg errgroup.Group) Option { + return func(r *router) { + r.eg = eg + } +} diff --git a/pkg/agent/core/faiss/router/router.go b/pkg/agent/core/faiss/router/router.go new file mode 100644 index 00000000000..4b3cb2cc83c --- /dev/null +++ b/pkg/agent/core/faiss/router/router.go @@ -0,0 +1,170 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package router provides implementation of Go API for routing http Handler wrapped by rest.Func +package router + +import ( + "net/http" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/net/http/middleware" + "github.com/vdaas/vald/internal/net/http/routing" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/rest" +) + +type router struct { + handler rest.Handler + eg errgroup.Group + timeout string +} + +// New returns REST route&method information from handler interface. +func New(opts ...Option) http.Handler { + r := new(router) + + for _, opt := range append(defaultOptions, opts...) { + opt(r) + } + + h := r.handler + + return routing.New( + routing.WithMiddleware( + middleware.NewTimeout( + middleware.WithTimeout(r.timeout), + middleware.WithErrorGroup(r.eg), + )), + routing.WithRoutes([]routing.Route{ + { + "Index", + []string{ + http.MethodGet, + }, + "/", + h.Index, + }, + { + "Search", + []string{ + http.MethodPost, + }, + "/search", + h.Search, + }, + { + "Search By ID", + []string{ + http.MethodPost, + }, + "/id/search", + h.SearchByID, + }, + { + "LinearSearch", + []string{ + http.MethodPost, + }, + "/linearsearch", + h.LinearSearch, + }, + { + "LinearSearch By ID", + []string{ + http.MethodPost, + }, + "/id/linearsearch", + h.LinearSearchByID, + }, + { + "Insert", + []string{ + http.MethodPost, + }, + "/insert", + h.Insert, + }, + { + "Multiple Insert", + []string{ + http.MethodPost, + }, + "/insert/multi", + h.MultiInsert, + }, + { + "Update", + []string{ + http.MethodPost, + http.MethodPatch, + http.MethodPut, + }, + "/update", + h.Update, + }, + { + "Multiple Update", + []string{ + http.MethodPost, + http.MethodPatch, + http.MethodPut, + }, + "/update/multi", + h.MultiUpdate, + }, + { + "Remove", + []string{ + http.MethodDelete, + }, + "/delete", + h.Remove, + }, + { + "Multiple Remove", + []string{ + http.MethodDelete, + http.MethodPost, + }, + "/delete/multi", + h.MultiRemove, + }, + { + "Create Index", + []string{ + http.MethodPost, + }, + "/index/create", + h.CreateIndex, + }, + { + "Save Index", + []string{ + http.MethodGet, + }, + "/index/save", + h.SaveIndex, + }, + { + "GetObject", + []string{ + http.MethodGet, + }, + "/object/{id}", + h.GetObject, + }, + }...)) +} diff --git a/pkg/agent/core/faiss/service/faiss.go b/pkg/agent/core/faiss/service/faiss.go new file mode 100644 index 00000000000..e81e63f9ee7 --- /dev/null +++ b/pkg/agent/core/faiss/service/faiss.go @@ -0,0 +1,1290 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package service manages the main logic of server. +package service + +import ( + "context" + "encoding/gob" + "io/fs" + "math" + "os" + "path/filepath" + "reflect" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/vdaas/vald/internal/config" + core "github.com/vdaas/vald/internal/core/algorithm/faiss" + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/pkg/agent/core/faiss/model" + "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" + "github.com/vdaas/vald/pkg/agent/core/ngt/service/vqueue" + "github.com/vdaas/vald/pkg/agent/internal/metadata" +) + +type ( + Faiss interface { + Start(ctx context.Context) <-chan error + Train(nb int, xb []float32) error + Insert(uuid string, xb []float32) error + InsertWithTime(uuid string, vec []float32, t int64) error + Update(uuid string, vec []float32) error + UpdateWithTime(uuid string, vec []float32, t int64) error + CreateIndex(ctx context.Context) error + SaveIndex(ctx context.Context) error + CreateAndSaveIndex(ctx context.Context) error + Search(k, nq uint32, xq []float32) ([]model.Distance, error) + Delete(uuid string) error + DeleteWithTime(uuid string, t int64) error + Exists(uuid string) (uint32, bool) + IsIndexing() bool + IsSaving() bool + NumberOfCreateIndexExecution() uint64 + NumberOfProactiveGCExecution() uint64 + Len() uint64 + InsertVQueueBufferLen() uint64 + DeleteVQueueBufferLen() uint64 + GetDimensionSize() int + GetTrainSize() int + Close(ctx context.Context) error + } + + faiss struct { + core core.Faiss + eg errgroup.Group + kvs kvs.BidiMap + fmu sync.Mutex + fmap map[string]int64 // failure map for index + vq vqueue.Queue + addVecs []float32 + addIds []int64 + isTrained bool + trainSize int + icnt uint64 + + // statuses + indexing atomic.Value + saving atomic.Value + cimu sync.Mutex // create index mutex + lastNocie uint64 // last number of create index execution this value prevent unnecessary saveindex + + // counters + nocie uint64 // number of create index execution + nogce uint64 // number of proactive GC execution + wfci uint64 // wait for create indexing + + // configurations + inMem bool // in-memory mode + dim int // dimension size + nlist int // the number of Voronoi cells + m int // number of subquantizers + alen int // auto indexing length + dur time.Duration // auto indexing check duration + sdur time.Duration // auto save index check duration + lim time.Duration // auto indexing time limit + minLit time.Duration // minimum load index timeout + maxLit time.Duration // maximum load index timeout + litFactor time.Duration // load index timeout factor + enableProactiveGC bool // if this value is true, agent component will purge GC memory more proactive + enableCopyOnWrite bool // if this value is true, agent component will write backup file using Copy on Write and saves old files to the old directory + path string // index path + smu sync.Mutex // save index lock + tmpPath atomic.Value // temporary index path for Copy on Write + oldPath string // old volume path + basePath string // index base directory for CoW + cowmu sync.Mutex // copy on write move lock + dcd bool // disable commit daemon + idelay time.Duration // initial delay duration + kvsdbConcurrency int // kvsdb concurrency + } +) + +const ( + kvsFileName = "faiss-meta.kvsdb" + kvsTimestampFileName = "faiss-timestamp.kvsdb" + noTimeStampFile = -1 + + oldIndexDirName = "backup" + originIndexDirName = "origin" + + // ref: https://github.com/facebookresearch/faiss/wiki/FAQ#can-i-ignore-warning-clustering-xxx-points-to-yyy-centroids + // ref: https://github.com/facebookresearch/faiss/blob/main/faiss/Clustering.cpp#L38 + minPointsPerCentroid int = 39 +) + +func New(cfg *config.Faiss, opts ...Option) (Faiss, error) { + var ( + f = &faiss{ + fmap: make(map[string]int64), + dim: cfg.Dimension, + nlist: cfg.Nlist, + m: cfg.M, + enableProactiveGC: cfg.EnableProactiveGC, + enableCopyOnWrite: cfg.EnableCopyOnWrite, + kvsdbConcurrency: cfg.KVSDB.Concurrency, + } + err error + ) + + for _, opt := range append(defaultOptions, opts...) { + if err := opt(f); err != nil { + return nil, errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + } + } + + if len(f.path) == 0 { + f.inMem = true + } + + if f.enableCopyOnWrite && !f.inMem && len(f.path) != 0 { + sep := string(os.PathSeparator) + f.path, err = filepath.Abs(strings.ReplaceAll(f.path, sep+sep, sep)) + if err != nil { + log.Warn(err) + } + + f.basePath = f.path + f.oldPath = file.Join(f.basePath, oldIndexDirName) + f.path = file.Join(f.basePath, originIndexDirName) + err = file.MkdirAll(f.oldPath, fs.ModePerm) + if err != nil { + log.Warn(err) + } + err = file.MkdirAll(f.path, fs.ModePerm) + if err != nil { + log.Warn(err) + } + err = f.mktmp() + if err != nil { + return nil, err + } + } + + err = f.initFaiss( + core.WithDimension(cfg.Dimension), + core.WithNlist(cfg.Nlist), + core.WithM(cfg.M), + core.WithNbitsPerIdx(cfg.NbitsPerIdx), + core.WithMetricType(cfg.MetricType), + ) + if err != nil { + return nil, err + } + + if f.dur == 0 || f.alen == 0 { + f.dcd = true + } + + if f.vq == nil { + f.vq, err = vqueue.New() + if err != nil { + return nil, err + } + } + + f.indexing.Store(false) + f.saving.Store(false) + + return f, nil +} + +func (f *faiss) initFaiss(opts ...core.Option) error { + var err error + + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + + if f.inMem { + log.Debug("vald agent starts with in-memory mode") + f.core, err = core.New(opts...) + return err + } + + ctx := context.Background() + err = f.load(ctx, f.path, opts...) + var current uint64 + if err != nil { + if !f.enableCopyOnWrite { + log.Debug("failed to load vald index from %s\t error: %v", f.path, err) + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } else if f.kvs.Len() > 0 { + f.kvs.Close() + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + + if f.core != nil { + f.core.Close() + f.core = nil + } + f.core, err = core.New(append(opts, core.WithIndexPath(f.path))...) + return err + } + + if errors.Is(err, errors.ErrIndicesAreTooFewComparedToMetadata) && f.kvs != nil { + current = f.kvs.Len() + log.Warnf( + "load vald primary index success from %s\t error: %v\tbut index data are too few %d compared to metadata count now trying to load from old copied index data from %s and compare them", + f.path, + err, + current, + f.oldPath, + ) + } else { + log.Warnf("failed to load vald primary index from %s\t error: %v\ttrying to load from old copied index data from %s", f.path, err, f.oldPath) + } + } else { + return nil + } + + err = f.load(ctx, f.oldPath, opts...) + if err == nil { + if current != 0 && f.kvs.Len() < current { + log.Warnf( + "load vald secondary index success from %s\t error: %v\tbut index data are too few %d compared to primary data now trying to load from primary index data again from %s and start up with them", + f.oldPath, + err, + f.kvs.Len(), + f.oldPath, + ) + + err = f.load(ctx, f.path, opts...) + if err == nil { + return nil + } + } else { + return nil + } + } + + log.Warnf("failed to load vald secondary index from %s and %s\t error: %v\ttrying to load from non-CoW index data from %s for backwards compatibility", f.path, f.oldPath, err, f.basePath) + err = f.load(ctx, f.basePath, opts...) + if err == nil { + file.CopyDir(ctx, f.basePath, f.path) + return nil + } + + tpath := f.tmpPath.Load().(string) + log.Warnf( + "failed to load vald backwards index from %s and %s and %s\t error: %v\tvald agent couldn't find any index from agent volume in %s trying to start as new index from %s", + f.path, + f.oldPath, + f.basePath, + err, + f.basePath, + tpath, + ) + + if f.core != nil { + f.core.Close() + f.core = nil + } + f.core, err = core.New(append(opts, core.WithIndexPath(tpath))...) + if err != nil { + return err + } + + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } else if f.kvs.Len() > 0 { + f.kvs.Close() + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + + return nil +} + +func (f *faiss) load(ctx context.Context, path string, opts ...core.Option) error { + exist, fi, err := file.ExistsWithDetail(path) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + err = errors.Wrapf(errors.ErrIndexFileNotFound, "index file does not exists,\tpath: %s,\terr: %v", path, err) + return err + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.Wrap(errors.ErrFailedToOpenFile(err, path, 0, fi.Mode()), "invalid permission for loading index path") + } + return err + case exist && fi != nil && fi.IsDir(): + if fi.Mode().IsDir() && !strings.HasSuffix(path, string(os.PathSeparator)) { + path += string(os.PathSeparator) + } + files, err := filepath.Glob(file.Join(filepath.Dir(path), "*")) + if err != nil || len(files) == 0 { + err = errors.Wrapf(errors.ErrIndexFileNotFound, "index path exists but no file does not exists in the directory,\tpath: %s,\tfiles: %v\terr: %v", path, files, err) + return err + } + if strings.HasSuffix(path, string(os.PathSeparator)) { + path = strings.TrimSuffix(path, string(os.PathSeparator)) + } + } + + metadataPath := file.Join(path, metadata.AgentMetadataFileName) + log.Debugf("index path: %s exists, now starting to check metadata from %s", path, metadataPath) + exist, fi, err = file.ExistsWithDetail(metadataPath) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + err = errors.Wrapf(errors.ErrIndexFileNotFound, "metadata file does not exists,\tpath: %s,\terr: %v", metadataPath, err) + return err + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.Wrap(errors.ErrFailedToOpenFile(err, metadataPath, 0, fi.Mode()), "invalid permission for loading metadata") + } + return err + } + + log.Debugf("index path: %s and metadata: %s exists, now starting to load metadata", path, metadataPath) + agentMetadata, err := metadata.Load(metadataPath) + if err != nil && errors.Is(err, fs.ErrNotExist) || agentMetadata == nil || agentMetadata.Faiss == nil || agentMetadata.Faiss.IndexCount == 0 { + err = errors.Wrapf(err, "cannot read metadata from path: %s\tmetadata: %s", path, agentMetadata) + return err + } + + kvsFilePath := file.Join(path, kvsFileName) + log.Debugf("index path: %s and metadata: %s exists and successfully load metadata, now starting to load kvs data from %s", path, metadataPath, kvsFilePath) + exist, fi, err = file.ExistsWithDetail(kvsFilePath) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + err = errors.Wrapf(errors.ErrIndexFileNotFound, "kvsdb file does not exists,\tpath: %s,\terr: %v", kvsFilePath, err) + return err + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.ErrFailedToOpenFile(err, kvsFilePath, 0, fi.Mode()) + } + err = errors.Wrapf(err, "invalid permission for loading kvsdb file from %s", kvsFilePath) + return err + } + + kvsTimestampFilePath := file.Join(path, kvsTimestampFileName) + log.Debugf("now starting to load kvs timestamp data from %s", kvsTimestampFilePath) + exist, fi, err = file.ExistsWithDetail(kvsTimestampFilePath) + switch { + case !exist, fi == nil, fi != nil && fi.Size() == 0, err != nil && errors.Is(err, fs.ErrNotExist): + log.Warnf("timestamp kvsdb file does not exists,\tpath: %s,\terr: %v", kvsTimestampFilePath, err) + case err != nil && errors.Is(err, fs.ErrPermission): + if fi != nil { + err = errors.ErrFailedToOpenFile(err, kvsTimestampFilePath, 0, fi.Mode()) + } + log.Warnf("invalid permission for loading timestamp kvsdb file from %s", kvsTimestampFilePath) + } + + var timeout time.Duration + if agentMetadata != nil && agentMetadata.Faiss != nil { + log.Debugf("the backup index size is %d. starting to load...", agentMetadata.Faiss.IndexCount) + timeout = time.Duration( + math.Min( + math.Max( + float64(agentMetadata.Faiss.IndexCount)*float64(f.litFactor), + float64(f.minLit), + ), + float64(f.maxLit), + ), + ) + } else { + log.Debugf("cannot inspect the backup index size. starting to load default value.") + timeout = time.Duration(math.Min(float64(f.minLit), float64(f.maxLit))) + } + + log.Debugf( + "index path: %s and metadata: %s and kvsdb file: %s and timestamp kvsdb file: %s exists and successfully load metadata, now starting to load full index and kvs data in concurrent", + path, + metadataPath, + kvsFilePath, + kvsTimestampFilePath, + ) + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + eg, _ := errgroup.New(ctx) + eg.Go(safety.RecoverFunc(func() (err error) { + if f.core != nil { + f.core.Close() + f.core = nil + } + f.core, err = core.Load(append(opts, core.WithIndexPath(path))...) + if err != nil { + err = errors.Wrapf(err, "failed to load faiss index from path: %s", path) + return err + } + return nil + })) + + eg.Go(safety.RecoverFunc(func() (err error) { + err = f.loadKVS(ctx, path, timeout) + if err != nil { + err = errors.Wrapf(err, "failed to load kvsdb data from path: %s, %s", kvsFilePath, kvsTimestampFilePath) + return err + } + if f.kvs != nil && float64(agentMetadata.Faiss.IndexCount/2) > float64(f.kvs.Len()) { + return errors.ErrIndicesAreTooFewComparedToMetadata + } + return nil + })) + + ech := make(chan error, 1) + // NOTE: when it exceeds the timeout while loading, + // it should exit this function and leave this goroutine running. + f.eg.Go(safety.RecoverFunc(func() error { + defer close(ech) + ech <- safety.RecoverFunc(func() (err error) { + err = eg.Wait() + if err != nil { + log.Error(err) + return err + } + cancel() + return nil + })() + return nil + })) + + select { + case err := <-ech: + return err + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + log.Errorf("cannot load index backup data from %s within the timeout %s. the process is going to be killed.", path, timeout) + err := metadata.Store(metadataPath, + &metadata.Metadata{ + IsInvalid: true, + Faiss: &metadata.Faiss{ + IndexCount: 0, + }, + }, + ) + if err != nil { + return err + } + return errors.ErrIndexLoadTimeout + } + } + + return nil +} + +func (f *faiss) loadKVS(ctx context.Context, path string, timeout time.Duration) (err error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + eg, _ := errgroup.New(ctx) + + m := make(map[string]uint32) + mt := make(map[string]int64) + + eg.Go(safety.RecoverFunc(func() (err error) { + gob.Register(map[string]uint32{}) + var fi *os.File + fi, err = file.Open( + file.Join(path, kvsFileName), + os.O_RDONLY|os.O_SYNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if fi != nil { + derr := fi.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + err = gob.NewDecoder(fi).Decode(&m) + if err != nil { + log.Errorf("error decoding kvsdb file,\terr: %v", err) + return err + } + return nil + })) + + eg.Go(safety.RecoverFunc(func() (err error) { + gob.Register(map[string]int64{}) + var ft *os.File + ft, err = file.Open( + file.Join(path, kvsTimestampFileName), + os.O_RDONLY|os.O_SYNC, + fs.ModePerm, + ) + if err != nil { + log.Warnf("error opening timestamp kvsdb file,\terr: %v", err) + } + defer func() { + if ft != nil { + derr := ft.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + err = gob.NewDecoder(ft).Decode(&mt) + if err != nil { + log.Warnf("error decoding timestamp kvsdb file,\terr: %v", err) + } + return nil + })) + + err = eg.Wait() + if err != nil { + return err + } + + if f.kvs == nil { + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } else if f.kvs.Len() > 0 { + f.kvs.Close() + f.kvs = kvs.New(kvs.WithConcurrency(f.kvsdbConcurrency)) + } + for k, id := range m { + if ts, ok := mt[k]; ok { + f.kvs.Set(k, id, ts) + } else { + // NOTE: SaveIndex do not write ngt-timestamp.kvsdb with timestamp 0. + f.kvs.Set(k, id, 0) + f.fmap[k] = int64(id) + } + } + for k := range mt { + if _, ok := m[k]; !ok { + f.fmap[k] = noTimeStampFile + } + } + + return nil +} + +func (f *faiss) mktmp() error { + if !f.enableCopyOnWrite { + return nil + } + + path, err := file.MkdirTemp(file.Join(os.TempDir(), "vald")) + if err != nil { + log.Warnf("failed to create temporary index file path directory %s:\terr: %v", path, err) + return err + } + + f.tmpPath.Store(path) + + return nil +} + +func (f *faiss) Start(ctx context.Context) <-chan error { + if f.dcd { + return nil + } + + ech := make(chan error, 2) + f.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + if f.dur <= 0 { + f.dur = math.MaxInt64 + } + if f.sdur <= 0 { + f.sdur = math.MaxInt64 + } + if f.lim <= 0 { + f.lim = math.MaxInt64 + } + + if f.idelay > 0 { + timer := time.NewTimer(f.idelay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + timer.Stop() + } + + tick := time.NewTicker(f.dur) + sTick := time.NewTicker(f.sdur) + limit := time.NewTicker(f.lim) + defer tick.Stop() + defer sTick.Stop() + defer limit.Stop() + for { + err = nil + select { + case <-ctx.Done(): + err = f.CreateIndex(ctx) + if err != nil && !errors.Is(err, errors.ErrUncommittedIndexNotFound) { + ech <- err + return errors.Wrap(ctx.Err(), err.Error()) + } + return ctx.Err() + case <-tick.C: + if f.vq.IVQLen() >= f.alen { + err = f.CreateIndex(ctx) + } + case <-limit.C: + err = f.CreateAndSaveIndex(ctx) + case <-sTick.C: + err = f.SaveIndex(ctx) + } + if err != nil && err != errors.ErrUncommittedIndexNotFound { + ech <- err + runtime.Gosched() + err = nil + } + } + })) + + return ech +} + +func (f *faiss) Train(nb int, xb []float32) error { + err := f.core.Train(nb, xb) + if err != nil { + log.Errorf("failed to faiss train", err) + return err + } + + return nil +} + +func (f *faiss) Insert(uuid string, vec []float32) error { + return f.insert(uuid, vec, time.Now().UnixNano(), true) +} + +func (f *faiss) InsertWithTime(uuid string, vec []float32, t int64) error { + if t <= 0 { + t = time.Now().UnixNano() + } + + return f.insert(uuid, vec, t, true) +} + +func (f *faiss) insert(uuid string, xb []float32, t int64, validation bool) error { + if len(uuid) == 0 { + err := errors.ErrUUIDNotFound(0) + return err + } + + if validation { + _, ok := f.Exists(uuid) + if ok { + return errors.ErrUUIDAlreadyExists(uuid) + } + } + + return f.vq.PushInsert(uuid, xb, t) +} + +func (f *faiss) Update(uuid string, vec []float32) error { + return f.update(uuid, vec, time.Now().UnixNano()) +} + +func (f *faiss) UpdateWithTime(uuid string, vec []float32, t int64) error { + if t <= 0 { + t = time.Now().UnixNano() + } + return f.update(uuid, vec, t) +} + +func (f *faiss) update(uuid string, vec []float32, t int64) (err error) { + if err = f.readyForUpdate(uuid, vec); err != nil { + return err + } + + err = f.delete(uuid, t, true) // `true` is to return NotFound error with non-existent ID + if err != nil { + return err + } + + t++ + return f.insert(uuid, vec, t, false) +} + +func (f *faiss) readyForUpdate(uuid string, vec []float32) (err error) { + if len(uuid) == 0 { + return errors.ErrUUIDNotFound(0) + } + + if len(vec) != f.GetDimensionSize() { + return errors.ErrInvalidDimensionSize(len(vec), f.GetDimensionSize()) + } + + // not impl GetObject() + + return nil +} + +func (f *faiss) CreateIndex(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "vald/agent-faiss/service/Faiss.CreateIndex") + defer func() { + if span != nil { + span.End() + } + }() + + ic := f.vq.IVQLen() + f.vq.DVQLen() + (len(f.addVecs) / f.dim) + if ic == 0 { + return errors.ErrUncommittedIndexNotFound + } + + wf := atomic.AddUint64(&f.wfci, 1) + if wf > 1 { + atomic.AddUint64(&f.wfci, ^uint64(0)) + log.Debugf("concurrent create index waiting detected this request will be ignored, concurrent: %d", wf) + return nil + } + + err := func() error { + ticker := time.NewTicker(time.Millisecond * 100) + defer ticker.Stop() + // wait for not indexing & not saving + for f.IsIndexing() || f.IsSaving() { + runtime.Gosched() + select { + case <-ctx.Done(): + atomic.AddUint64(&f.wfci, ^uint64(0)) + return ctx.Err() + case <-ticker.C: + } + } + atomic.AddUint64(&f.wfci, ^uint64(0)) + return nil + }() + if err != nil { + return err + } + + f.cimu.Lock() + defer f.cimu.Unlock() + f.indexing.Store(true) + defer f.indexing.Store(false) + defer f.gc() + now := time.Now().UnixNano() + ic = f.vq.IVQLen() + f.vq.DVQLen() + (len(f.addVecs) / f.dim) + if ic == 0 { + return errors.ErrUncommittedIndexNotFound + } + + log.Infof("create index operation started, uncommitted indexes = %d", ic) + log.Debug("create index delete phase started") + f.vq.RangePopDelete(ctx, now, func(uuid string) bool { + log.Debugf("start delete operation for kvsdb id: %s", uuid) + oid, ok := f.kvs.Delete(uuid) + if !ok { + log.Warn(errors.ErrObjectIDNotFound(uuid)) + return true + } + log.Debugf("start remove operation for faiss index id: %s, oid: %d", uuid, oid) + ntotal, err := f.core.Remove(1, []int64{int64(oid)}) + if err != nil { + log.Errorf("failed to remove oid: %d from faiss index. error: %v", oid, err) + f.fmu.Lock() + f.fmap[uuid] = int64(oid) + f.fmu.Unlock() + } + log.Debugf("removed from faiss index and kvsdb id: %s, oid: %d, index size: %d", uuid, oid, ntotal) + return true + }) + log.Debug("create index delete phase finished") + + f.gc() + + log.Debug("create index insert phase started") + f.vq.RangePopInsert(ctx, now, func(uuid string, vector []float32, timestamp int64) bool { + log.Debugf("start stack operation for faiss index id: %s, icnt: %d", uuid, uint32(f.icnt)) + f.addVecs = append(f.addVecs, vector...) + f.addIds = append(f.addIds, int64(f.icnt)) + + log.Debugf("start insert operation for kvsdb id: %s, icnt: %d", uuid, uint32(f.icnt)) + f.kvs.Set(uuid, uint32(f.icnt), timestamp) + atomic.AddUint64(&f.icnt, 1) + + f.fmu.Lock() + _, ok := f.fmap[uuid] + if ok { + delete(f.fmap, uuid) + } + f.fmu.Unlock() + log.Debugf("finished to insert index and kvsdb id: %s, icnt: %d", uuid, uint32(f.icnt)) + return true + }) + + var max int + if f.nlist > int(math.Pow(2, float64(f.m))) { + max = f.nlist + } else { + max = int(math.Pow(2, float64(f.m))) + } + if !f.isTrained && len(f.addVecs)/f.dim >= max*minPointsPerCentroid { + log.Debug("faiss train phase started") + log.Debugf("max * minPointsPerCentroid: %d", max*minPointsPerCentroid) + err := f.core.Train(len(f.addVecs)/f.dim, f.addVecs) + if err != nil { + log.Errorf("failed to faiss train", err) + return err + } + f.isTrained = true + f.trainSize = len(f.addVecs) / f.dim + log.Debug("faiss train phase finished") + } + if f.isTrained && len(f.addVecs) > 0 { + log.Debug("faiss add phase started") + ntotal, err := f.core.Add(len(f.addVecs)/f.dim, f.addVecs, f.addIds) + if err != nil { + log.Errorf("failed to faiss add", err) + return err + } + f.addVecs = nil + f.addIds = nil + log.Debugf("is trained: %v, index size: %d", f.isTrained, ntotal) + log.Debug("faiss add phase finished") + } + log.Debug("create index insert phase finished") + + atomic.AddUint64(&f.nocie, 1) + log.Info("create index operation finished") + + return nil +} + +func (f *faiss) SaveIndex(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "vald/agent-faiss/service/Faiss.SaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + + if !f.inMem { + return f.saveIndex(ctx) + } + + return nil +} + +func (f *faiss) saveIndex(ctx context.Context) error { + nocie := atomic.LoadUint64(&f.nocie) + if atomic.LoadUint64(&f.lastNocie) == nocie || !f.isTrained { + return nil + } + atomic.SwapUint64(&f.lastNocie, nocie) + + err := func() error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + // wait for not indexing & not saving + for f.IsIndexing() || f.IsSaving() { + runtime.Gosched() + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + return nil + }() + if err != nil { + return err + } + + f.saving.Store(true) + defer f.gc() + defer f.saving.Store(false) + + // no cleanup invalid index + + eg, ectx := errgroup.New(ctx) + // we want to ensure the acutal kvs size between kvsdb and metadata, + // so we create this counter to count the actual kvs size instead of using kvs.Len() + var ( + kvsLen uint64 + path string + ) + + if f.enableCopyOnWrite { + path = f.tmpPath.Load().(string) + } else { + path = f.path + } + + f.smu.Lock() + defer f.smu.Unlock() + + eg.Go(safety.RecoverFunc(func() (err error) { + if f.kvs.Len() > 0 && path != "" { + m := make(map[string]uint32, f.Len()) + mt := make(map[string]int64, f.Len()) + var mu sync.Mutex + + f.kvs.Range(ectx, func(key string, id uint32, ts int64) bool { + mu.Lock() + m[key] = id + mt[key] = ts + mu.Unlock() + atomic.AddUint64(&kvsLen, 1) + return true + }) + + var fi *os.File + fi, err = file.Open( + file.Join(path, kvsFileName), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if fi != nil { + derr := fi.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + + gob.Register(map[string]uint32{}) + err = gob.NewEncoder(fi).Encode(&m) + if err != nil { + return err + } + + err = fi.Sync() + if err != nil { + return err + } + + m = make(map[string]uint32) + + var ft *os.File + ft, err = file.Open( + file.Join(path, kvsTimestampFileName), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if ft != nil { + derr := ft.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + + gob.Register(map[string]int64{}) + err = gob.NewEncoder(ft).Encode(&mt) + if err != nil { + return err + } + + err = ft.Sync() + if err != nil { + return err + } + + mt = make(map[string]int64) + } + + return nil + })) + + eg.Go(safety.RecoverFunc(func() (err error) { + f.fmu.Lock() + fl := len(f.fmap) + f.fmu.Unlock() + + if fl > 0 && path != "" { + var fi *os.File + fi, err = file.Open( + file.Join(path, "invalid-"+kvsFileName), + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + fs.ModePerm, + ) + if err != nil { + return err + } + defer func() { + if fi != nil { + derr := fi.Close() + if derr != nil { + err = errors.Wrap(err, derr.Error()) + } + } + }() + + gob.Register(map[string]int64{}) + f.fmu.Lock() + err = gob.NewEncoder(fi).Encode(&f.fmap) + f.fmu.Unlock() + if err != nil { + return err + } + + err = fi.Sync() + if err != nil { + return err + } + } + + return nil + })) + + eg.Go(safety.RecoverFunc(func() error { + return f.core.SaveIndexWithPath(path) + })) + + err = eg.Wait() + if err != nil { + return err + } + + err = metadata.Store( + file.Join(path, metadata.AgentMetadataFileName), + &metadata.Metadata{ + IsInvalid: false, + Faiss: &metadata.Faiss{ + IndexCount: kvsLen, + }, + }, + ) + if err != nil { + return err + } + + return f.moveAndSwitchSavedData(ctx) +} + +func (f *faiss) moveAndSwitchSavedData(ctx context.Context) error { + if !f.enableCopyOnWrite { + return nil + } + + var err error + f.cowmu.Lock() + defer f.cowmu.Unlock() + + err = file.MoveDir(ctx, f.path, f.oldPath) + if err != nil { + log.Warnf("failed to backup backup data from %s to %s error: %v", f.path, f.oldPath, err) + } + + path := f.tmpPath.Load().(string) + err = file.MoveDir(ctx, path, f.path) + if err != nil { + log.Warnf("failed to move temporary index data from %s to %s error: %v, trying to rollback secondary backup data from %s to %s", path, f.path, f.oldPath, f.path, err) + return file.MoveDir(ctx, f.oldPath, f.path) + } + defer log.Warnf("finished to copy index from %s => %s => %s", path, f.path, f.oldPath) + + return f.mktmp() +} + +func (f *faiss) CreateAndSaveIndex(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, "vald/agent-faiss/service/Faiss.CreateAndSaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + + err := f.CreateIndex(ctx) + if err != nil && + !errors.Is(err, errors.ErrUncommittedIndexNotFound) && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + return err + } + + return f.SaveIndex(ctx) +} + +func (f *faiss) Search(k, nq uint32, xq []float32) ([]model.Distance, error) { + if f.IsIndexing() { + return nil, errors.ErrCreateIndexingIsInProgress + } + + sr, err := f.core.Search(int(k), int(nq), xq) + if err != nil { + if f.IsIndexing() { + return nil, errors.ErrCreateIndexingIsInProgress + } + + log.Errorf("cgo error detected: faiss api returned error %v", err) + return nil, err + } + + if len(sr) == 0 { + return nil, errors.ErrEmptySearchResult + } + + ds := make([]model.Distance, 0, len(sr)) + for _, d := range sr { + if err = d.Error; d.ID == 0 && err != nil { + log.Warnf("an error occurred while searching: %s", err) + continue + } + + key, _, ok := f.kvs.GetInverse(d.ID) + if ok { + ds = append(ds, model.Distance{ + ID: key, + Distance: d.Distance, + }) + } else { + log.Warn("not found", d.ID, d.Distance) + } + } + + return ds, nil +} + +func (f *faiss) Delete(uuid string) (err error) { + return f.delete(uuid, time.Now().UnixNano(), true) +} + +func (f *faiss) DeleteWithTime(uuid string, t int64) (err error) { + if t <= 0 { + t = time.Now().UnixNano() + } + + return f.delete(uuid, t, true) +} + +func (f *faiss) delete(uuid string, t int64, validation bool) error { + if len(uuid) == 0 { + return errors.ErrUUIDNotFound(0) + } + + if validation { + _, _, ok := f.kvs.Get(uuid) + if !ok && !f.vq.IVExists(uuid) { + return errors.ErrObjectIDNotFound(uuid) + } + } + + return f.vq.PushDelete(uuid, t) +} + +func (f *faiss) Exists(uuid string) (uint32, bool) { + var ( + oid uint32 + ok bool + ) + + ok = f.vq.IVExists(uuid) + if !ok { + oid, _, ok = f.kvs.Get(uuid) + if !ok { + log.Debugf("Exists\tuuid: %s's data not found in kvsdb and insert vqueue\terror: %v", uuid, errors.ErrObjectIDNotFound(uuid)) + return 0, false + } + if f.vq.DVExists(uuid) { + log.Debugf("Exists\tuuid: %s's data found in kvsdb and not found in insert vqueue, but delete vqueue data exists. the object will be delete soon\terror: %v", + uuid, errors.ErrObjectIDNotFound(uuid)) + return 0, false + } + } + + return oid, ok +} + +func (f *faiss) IsIndexing() bool { + i, ok := f.indexing.Load().(bool) + return i && ok +} + +func (f *faiss) IsSaving() bool { + s, ok := f.saving.Load().(bool) + return s && ok +} + +func (f *faiss) NumberOfCreateIndexExecution() uint64 { + return atomic.LoadUint64(&f.nocie) +} + +func (f *faiss) NumberOfProactiveGCExecution() uint64 { + return atomic.LoadUint64(&f.nogce) +} + +func (f *faiss) gc() { + if f.enableProactiveGC { + runtime.GC() + atomic.AddUint64(&f.nogce, 1) + } +} + +func (f *faiss) Len() uint64 { + return f.kvs.Len() +} + +func (f *faiss) InsertVQueueBufferLen() uint64 { + return uint64(f.vq.IVQLen()) +} + +func (f *faiss) DeleteVQueueBufferLen() uint64 { + return uint64(f.vq.DVQLen()) +} + +func (f *faiss) GetDimensionSize() int { + return f.dim +} + +func (f *faiss) GetTrainSize() int { + return f.trainSize +} + +func (f *faiss) Close(ctx context.Context) error { + err := f.kvs.Close() + if len(f.path) != 0 { + cerr := f.CreateIndex(ctx) + if cerr != nil && + !errors.Is(err, errors.ErrUncommittedIndexNotFound) && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + if err != nil { + err = errors.Wrap(cerr, err.Error()) + } else { + err = cerr + } + } + + serr := f.SaveIndex(ctx) + if serr != nil && + !errors.Is(err, errors.ErrUncommittedIndexNotFound) && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + if err != nil { + err = errors.Wrap(serr, err.Error()) + } else { + err = serr + } + } + } + + f.core.Close() + + return nil +} diff --git a/pkg/agent/core/faiss/service/option.go b/pkg/agent/core/faiss/service/option.go new file mode 100644 index 00000000000..1d271dffdc0 --- /dev/null +++ b/pkg/agent/core/faiss/service/option.go @@ -0,0 +1,271 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "math" + "math/big" + "os" + "time" + + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/rand" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/timeutil" +) + +// Option represent the functional option for faiss +type Option func(f *faiss) error + +var defaultOptions = []Option{ + WithErrGroup(errgroup.Get()), + WithAutoIndexCheckDuration("30m"), + WithAutoSaveIndexDuration("35m"), + WithAutoIndexDurationLimit("24h"), + WithAutoIndexLength(100), + WithInitialDelayMaxDuration("3m"), + WithMinLoadIndexTimeout("3m"), + WithMaxLoadIndexTimeout("10m"), + WithLoadIndexTimeoutFactor("1ms"), + WithProactiveGC(true), +} + +// WithErrGroup returns the functional option to set the error group. +func WithErrGroup(eg errgroup.Group) Option { + return func(f *faiss) error { + if eg != nil { + f.eg = eg + } + + return nil + } +} + +// WithEnableInMemoryMode returns the functional option to set the in memory mode flag. +func WithEnableInMemoryMode(enabled bool) Option { + return func(f *faiss) error { + f.inMem = enabled + + return nil + } +} + +// WithIndexPath returns the functional option to set the index path of the Faiss. +func WithIndexPath(path string) Option { + return func(f *faiss) error { + if path == "" { + return nil + } + f.path = file.Join(strings.TrimSuffix(path, string(os.PathSeparator))) + return nil + } +} + +// WithAutoIndexCheckDuration returns the functional option to set the index check duration. +func WithAutoIndexCheckDuration(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.dur = d + + return nil + } +} + +// WithAutoSaveIndexDuration returns the functional option to set the auto save index duration. +func WithAutoSaveIndexDuration(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.sdur = d + + return nil + } +} + +// WithAutoIndexDurationLimit returns the functional option to set the auto index duration limit. +func WithAutoIndexDurationLimit(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.lim = d + + return nil + } +} + +// WithAutoIndexLength returns the functional option to set the auto index length. +func WithAutoIndexLength(l int) Option { + return func(f *faiss) error { + f.alen = l + + return nil + } +} + +const ( + defaultDurationLimit float64 = 1.1 + defaultRandDuration int64 = 1 +) + +var ( + bigMaxFloat64 = big.NewFloat(math.MaxFloat64) + bigMinFloat64 = big.NewFloat(math.SmallestNonzeroFloat64) + bigMaxInt64 = big.NewInt(math.MaxInt64) + bigMinInt64 = big.NewInt(math.MinInt64) +) + +// WithInitialDelayMaxDuration returns the functional option to set the initial delay duration. +func WithInitialDelayMaxDuration(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + var dt time.Duration + switch { + case d <= time.Nanosecond: + return nil + case d <= time.Microsecond: + dt = time.Nanosecond + case d <= time.Millisecond: + dt = time.Microsecond + case d <= time.Second: + dt = time.Millisecond + default: + dt = time.Second + } + + dbs := math.Round(float64(d) / float64(dt)) + bdbs := big.NewFloat(dbs) + if dbs <= 0 || bigMaxFloat64.Cmp(bdbs) <= 0 || bigMinFloat64.Cmp(bdbs) >= 0 { + dbs = defaultDurationLimit + } + + rnd := int64(rand.LimitedUint32(uint64(dbs))) + brnd := big.NewInt(rnd) + if rnd <= 0 || bigMaxInt64.Cmp(brnd) <= 0 || bigMinInt64.Cmp(brnd) >= 0 { + rnd = defaultRandDuration + } + + delay := time.Duration(rnd) * dt + if delay <= 0 || delay >= math.MaxInt64 || delay <= math.MinInt64 { + return WithInitialDelayMaxDuration(dur)(f) + } + + f.idelay = delay + + return nil + } +} + +// WithMinLoadIndexTimeout returns the functional option to set the minimal load index timeout. +func WithMinLoadIndexTimeout(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.minLit = d + + return nil + } +} + +// WithMaxLoadIndexTimeout returns the functional option to set the maximum load index timeout. +func WithMaxLoadIndexTimeout(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.maxLit = d + + return nil + } +} + +// WithLoadIndexTimeoutFactor returns the functional option to set the factor of load index timeout. +func WithLoadIndexTimeoutFactor(dur string) Option { + return func(f *faiss) error { + if dur == "" { + return nil + } + + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + + f.litFactor = d + + return nil + } +} + +// WithProactiveGC returns the functional option to set the proactive GC enable flag. +func WithProactiveGC(enabled bool) Option { + return func(f *faiss) error { + f.enableProactiveGC = enabled + return nil + } +} + +// WithCopyOnWrite returns the functional option to set the CoW enable flag. +func WithCopyOnWrite(enabled bool) Option { + return func(f *faiss) error { + f.enableCopyOnWrite = enabled + return nil + } +} diff --git a/pkg/agent/core/faiss/usecase/agentd.go b/pkg/agent/core/faiss/usecase/agentd.go new file mode 100644 index 00000000000..12dcc430ad8 --- /dev/null +++ b/pkg/agent/core/faiss/usecase/agentd.go @@ -0,0 +1,193 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package usecase + +import ( + "context" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + vald "github.com/vdaas/vald/apis/grpc/v1/vald" + iconf "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/observability" + faissmetrics "github.com/vdaas/vald/internal/observability/metrics/agent/core/faiss" + infometrics "github.com/vdaas/vald/internal/observability/metrics/info" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/pkg/agent/core/faiss/config" + handler "github.com/vdaas/vald/pkg/agent/core/faiss/handler/grpc" + "github.com/vdaas/vald/pkg/agent/core/faiss/handler/rest" + "github.com/vdaas/vald/pkg/agent/core/faiss/router" + "github.com/vdaas/vald/pkg/agent/core/faiss/service" +) + +type run struct { + eg errgroup.Group + cfg *config.Data + faiss service.Faiss + server starter.Server + observability observability.Observability +} + +func New(cfg *config.Data) (r runner.Runner, err error) { + faiss, err := service.New( + cfg.Faiss, + service.WithErrGroup(errgroup.Get()), + service.WithEnableInMemoryMode(cfg.Faiss.EnableInMemoryMode), + service.WithIndexPath(cfg.Faiss.IndexPath), + service.WithAutoIndexCheckDuration(cfg.Faiss.AutoIndexCheckDuration), + service.WithAutoSaveIndexDuration(cfg.Faiss.AutoSaveIndexDuration), + service.WithAutoIndexDurationLimit(cfg.Faiss.AutoIndexDurationLimit), + service.WithAutoIndexLength(cfg.Faiss.AutoIndexLength), + service.WithInitialDelayMaxDuration(cfg.Faiss.InitialDelayMaxDuration), + service.WithMinLoadIndexTimeout(cfg.Faiss.MinLoadIndexTimeout), + service.WithMaxLoadIndexTimeout(cfg.Faiss.MaxLoadIndexTimeout), + service.WithLoadIndexTimeoutFactor(cfg.Faiss.LoadIndexTimeoutFactor), + service.WithProactiveGC(cfg.Faiss.EnableProactiveGC), + service.WithCopyOnWrite(cfg.Faiss.EnableCopyOnWrite), + ) + if err != nil { + return nil, err + } + + g, err := handler.New( + handler.WithFaiss(faiss), + handler.WithStreamConcurrency(cfg.Server.GetGRPCStreamConcurrency()), + ) + if err != nil { + return nil, err + } + + eg := errgroup.Get() + + grpcServerOptions := []server.Option{ + server.WithGRPCRegistFunc(func(srv *grpc.Server) { + agent.RegisterAgentServer(srv, g) + vald.RegisterValdServer(srv, g) + }), + server.WithPreStartFunc(func() error { + return nil + }), + server.WithPreStopFunction(func() error { + return nil + }), + } + + var obs observability.Observability + if cfg.Observability != nil && cfg.Observability.Enabled { + obs, err = observability.NewWithConfig( + cfg.Observability, + faissmetrics.New(faiss), + infometrics.New("agent_core_faiss_info", "Agent Faiss info", *cfg.Faiss), + ) + if err != nil { + return nil, err + } + } + + srv, err := starter.New( + starter.WithConfig(cfg.Server), + starter.WithREST(func(sc *iconf.Server) []server.Option { + return []server.Option{ + server.WithHTTPHandler( + router.New( + router.WithTimeout(sc.HTTP.HandlerTimeout), + router.WithErrGroup(eg), + router.WithHandler( + rest.New( + rest.WithAgent(g), + ), + ), + ), + ), + } + }), + starter.WithGRPC(func(sc *iconf.Server) []server.Option { + return grpcServerOptions + }), + ) + if err != nil { + return nil, err + } + + return &run{ + eg: eg, + faiss: faiss, + cfg: cfg, + server: srv, + observability: obs, + }, nil +} + +func (r *run) PreStart(ctx context.Context) error { + if r.observability != nil { + return r.observability.PreStart(ctx) + } + + return nil +} + +func (r *run) Start(ctx context.Context) (<-chan error, error) { + ech := make(chan error, 3) + var oech, nech, sech <-chan error + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + if r.observability != nil { + oech = r.observability.Start(ctx) + } + nech = r.faiss.Start(ctx) + sech = r.server.ListenAndServe(ctx) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-oech: + case err = <-nech: + case err = <-sech: + } + if err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + case ech <- err: + } + } + } + })) + + return ech, nil +} + +func (r *run) PreStop(ctx context.Context) error { + return nil +} + +func (r *run) Stop(ctx context.Context) error { + if r.observability != nil { + r.observability.Stop(ctx) + } + + return r.server.Shutdown(ctx) +} + +func (r *run) PostStop(ctx context.Context) error { + r.faiss.Close(ctx) + return nil +} diff --git a/pkg/agent/internal/metadata/metadata.go b/pkg/agent/internal/metadata/metadata.go index 4c0225f98d2..3b359ad283f 100644 --- a/pkg/agent/internal/metadata/metadata.go +++ b/pkg/agent/internal/metadata/metadata.go @@ -32,14 +32,19 @@ const ( ) type Metadata struct { - IsInvalid bool `json:"is_invalid" yaml:"is_invalid"` - NGT *NGT `json:"ngt,omitempty" yaml:"ngt"` + IsInvalid bool `json:"is_invalid" yaml:"is_invalid"` + NGT *NGT `json:"ngt,omitempty" yaml:"ngt"` + Faiss *Faiss `json:"faiss,omitempty" yaml:"faiss"` } type NGT struct { IndexCount uint64 `json:"index_count" yaml:"index_count"` } +type Faiss struct { + IndexCount uint64 `json:"index_count" yaml:"index_count"` +} + func Load(path string) (meta *Metadata, err error) { var fi os.FileInfo exists, fi, err := file.ExistsWithDetail(path) diff --git a/tests/e2e/crud/crud_faiss_test.go b/tests/e2e/crud/crud_faiss_test.go new file mode 100644 index 00000000000..548738a61ab --- /dev/null +++ b/tests/e2e/crud/crud_faiss_test.go @@ -0,0 +1,331 @@ +//go:build e2e + +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// package crud provides e2e tests using ann-benchmarks datasets +package crud + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/tests/e2e/hdf5" + "github.com/vdaas/vald/tests/e2e/kubernetes/client" + "github.com/vdaas/vald/tests/e2e/kubernetes/portforward" + "github.com/vdaas/vald/tests/e2e/operation" +) + +var ( + host string + port int + ds *hdf5.Dataset + + insertNum int + searchNum int + searchByIDNum int + getObjectNum int + updateNum int + upsertNum int + removeNum int + + insertFrom int + searchFrom int + searchByIDFrom int + getObjectFrom int + updateFrom int + upsertFrom int + removeFrom int + + waitAfterInsertDuration time.Duration + + kubeClient client.Client + namespace string + + forwarder *portforward.Portforward +) + +func init() { + testing.Init() + + flag.StringVar(&host, "host", "localhost", "hostname") + flag.IntVar(&port, "port", 8081, "gRPC port") + + flag.IntVar(&insertNum, "insert-num", 10000, "number of id-vector pairs used for insert") + flag.IntVar(&searchNum, "search-num", 10000, "number of id-vector pairs used for search") + flag.IntVar(&updateNum, "update-num", 10000, "number of id-vector pairs used for update") + flag.IntVar(&removeNum, "remove-num", 10000, "number of id-vector pairs used for remove") + + flag.IntVar(&insertFrom, "insert-from", 0, "first index of id-vector pairs used for insert") + flag.IntVar(&searchFrom, "search-from", 0, "first index of id-vector pairs used for search") + flag.IntVar(&updateFrom, "update-from", 0, "first index of id-vector pairs used for update") + flag.IntVar(&removeFrom, "remove-from", 0, "first index of id-vector pairs used for remove") + + datasetName := flag.String("dataset", "fashion-mnist-784-euclidean.hdf5", "dataset") + waitAfterInsert := flag.String("wait-after-insert", "3m", "wait duration after inserting vectors") + + pf := flag.Bool("portforward", false, "enable port forwarding") + pfPodName := flag.String("portforward-pod-name", "vald-gateway-0", "pod name (only for port forward)") + pfPodPort := flag.Int("portforward-pod-port", port, "pod gRPC port (only for port forward)") + + kubeConfig := flag.String("kubeconfig", file.Join(os.Getenv("HOME"), ".kube", "config"), "kubeconfig path") + flag.StringVar(&namespace, "namespace", "default", "namespace") + + flag.Parse() + + var err error + if *pf { + kubeClient, err = client.New(*kubeConfig) + if err != nil { + panic(err) + } + + forwarder = kubeClient.Portforward(namespace, *pfPodName, port, *pfPodPort) + + err = forwarder.Start() + if err != nil { + panic(err) + } + } + + fmt.Printf("loading dataset: %s ", *datasetName) + ds, err = hdf5.HDF5ToDataset(*datasetName) + if err != nil { + panic(err) + } + fmt.Println("loading finished") + + waitAfterInsertDuration, err = time.ParseDuration(*waitAfterInsert) + if err != nil { + panic(err) + } +} + +func teardown() { + if forwarder != nil { + forwarder.Close() + } +} + +func sleep(t *testing.T, dur time.Duration) { + t.Logf("%v sleep for %s.", time.Now(), dur) + time.Sleep(dur) + t.Logf("%v sleep finished.", time.Now()) +} + +func TestE2EInsertOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Insert(t, ctx, operation.Dataset{ + Train: ds.Train[insertFrom : insertFrom+insertNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2ESearchOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Search(t, ctx, operation.Dataset{ + Test: ds.Test[searchFrom : searchFrom+searchNum], + Neighbors: ds.Neighbors[searchFrom : searchFrom+searchNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2EUpdateOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.UpdateWithParameters( + t, + ctx, + operation.Dataset{ + Train: ds.Train[updateFrom : updateFrom+updateNum], + }, + true, + 1, + func(t *testing.T, status int32, msg string) error { + t.Helper() + + if status != int32(codes.NotFound) { + return errors.Errorf("the returned status is not NotFound on Update #1: %s", err) + } + + t.Logf("received a NotFound error on #1: %s", msg) + + return nil + }, + func(t *testing.T, err error) error { + t.Helper() + + st, _, _ := status.ParseError(err, codes.Unknown, "") + if st.Code() != codes.NotFound { + return err + } + + return nil + }, + ) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2ERemoveOnly(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Remove(t, ctx, operation.Dataset{ + Train: ds.Train[removeFrom : removeFrom+removeNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2EInsertAndSearch(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Insert(t, ctx, operation.Dataset{ + Train: ds.Train[insertFrom : insertFrom+insertNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + sleep(t, waitAfterInsertDuration) + + err = op.Search(t, ctx, operation.Dataset{ + Test: ds.Test[searchFrom : searchFrom+searchNum], + Neighbors: ds.Neighbors[searchFrom : searchFrom+searchNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} + +func TestE2EStandardCRUD(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Insert(t, ctx, operation.Dataset{ + Train: ds.Train[insertFrom : insertFrom+insertNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + sleep(t, waitAfterInsertDuration) + + err = op.Search(t, ctx, operation.Dataset{ + Test: ds.Test[searchFrom : searchFrom+searchNum], + Neighbors: ds.Neighbors[searchFrom : searchFrom+searchNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Exists(t, ctx, "0") + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.UpdateWithParameters( + t, + ctx, + operation.Dataset{ + Train: ds.Train[updateFrom : updateFrom+updateNum], + }, + true, + 1, + func(t *testing.T, status int32, msg string) error { + t.Helper() + + if status != int32(codes.NotFound) { + return errors.Errorf("the returned status is not NotFound on Update #1: %s", err) + } + + t.Logf("received a NotFound error on #1: %s", msg) + + return nil + }, + func(t *testing.T, err error) error { + t.Helper() + + st, _, _ := status.ParseError(err, codes.Unknown, "") + if st.Code() != codes.NotFound { + return err + } + + return nil + }, + ) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + err = op.Remove(t, ctx, operation.Dataset{ + Train: ds.Train[removeFrom : removeFrom+removeNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} diff --git a/versions/FAISS_VERSION b/versions/FAISS_VERSION new file mode 100644 index 00000000000..10c088013f8 --- /dev/null +++ b/versions/FAISS_VERSION @@ -0,0 +1 @@ +1.7.4 diff --git a/versions/NGT_VERSION b/versions/NGT_VERSION index 82bd22f9cc3..3d45b5c65a6 100644 --- a/versions/NGT_VERSION +++ b/versions/NGT_VERSION @@ -1 +1 @@ -2.0.13 +2.0.14 diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index c734575a828..c229fa5cbb1 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1 @@ -47.1.0 +47.2.0