diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index bcded1a18c..c01846873d 100755 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -22,7 +22,7 @@ assignees: "" -- Go Version: 1.21.1 +- Go Version: 1.21.3 - Docker Version: 20.10.8 - Kubernetes Version: v1.28.2 - NGT Version: 2.1.3 diff --git a/.github/ISSUE_TEMPLATE/security_issue_report.md b/.github/ISSUE_TEMPLATE/security_issue_report.md index 6f69df2df9..987648847c 100644 --- a/.github/ISSUE_TEMPLATE/security_issue_report.md +++ b/.github/ISSUE_TEMPLATE/security_issue_report.md @@ -16,7 +16,7 @@ assignees: "" -- Go Version: 1.21.1 +- Go Version: 1.21.3 - Docker Version: 20.10.8 - Kubernetes Version: v1.28.2 - NGT Version: 2.1.3 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2788d2b96c..0d9c72d881 100755 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,7 +16,7 @@ -- Go Version: 1.21.1 +- Go Version: 1.21.3 - Docker Version: 20.10.8 - Kubernetes Version: v1.28.2 - NGT Version: 2.1.3 diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index 897f333926..0a4e51fb8f 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -81,7 +81,6 @@ runs: with: agents: 3 ingress_port: ${{ inputs.ingress_port }} - options: "--image docker.io/rancher/k3s:latest" - name: Check Kubernetes cluster shell: bash run: | diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml index 079def0631..99e10bfa27 100644 --- a/.github/actions/setup-k3d/action.yaml +++ b/.github/actions/setup-k3d/action.yaml @@ -21,6 +21,10 @@ inputs: description: "k3d version" required: false default: "latest" + k3s_version: + description: "The k3s to use. The default version is `versions/K3S_VERSION`" + required: false + default: "" name: description: "Cluster name" required: false @@ -52,6 +56,16 @@ runs: env: K3D_VERSION: ${{ inputs.version }} + - name: Detect k3s version + id: k3s_version + shell: bash + run: | + K3S_VERSION=${K3S_VERSION:-`cat versions/K3S_VERSION`} + + echo "tag=${K3S_VERSION=$}" >> $GITHUB_OUTPUT + env: + K3S_VERSION: ${{ inputs.k3s_version }} + - name: Install k3d shell: bash run: | @@ -74,11 +88,13 @@ runs: if [ "${INGRESS_PORT}" != 0 ]; then OPTIONS="${OPTIONS} -p ${INGRESS_PORT}:80@loadbalancer" fi + OPTIONS="${OPTIONS} --image rancher/k3s:${K3S_VERSION}" echo "options=${OPTIONS}" >> $GITHUB_OUTPUT env: AGENTS: ${{ inputs.agents }} INGRESS_PORT: ${{ inputs.ingress_port }} OPTIONS: ${{ inputs.options }} + K3S_VERSION: ${{ steps.k3s_version.outputs.tag }} - name: Create k8s cluster shell: bash diff --git a/.github/helm/values/values-lb.yaml b/.github/helm/values/values-lb.yaml index 591a269672..7075b90708 100644 --- a/.github/helm/values/values-lb.yaml +++ b/.github/helm/values/values-lb.yaml @@ -69,3 +69,9 @@ manager: auto_index_duration_limit: 2m auto_index_check_duration: 30s auto_index_length: 1000 + corrector: + enabled: true + # suspend because you do not want corrector to start automatically in CI + # instead run it manually + suspend: true + schedule: "1 2 3 4 5" diff --git a/.github/valdrelease/valdrelease.yaml b/.github/valdrelease/valdrelease.yaml index d2df3e28d1..e49752f4b4 100644 --- a/.github/valdrelease/valdrelease.yaml +++ b/.github/valdrelease/valdrelease.yaml @@ -76,3 +76,7 @@ spec: auto_index_duration_limit: 2m auto_index_check_duration: 30s auto_index_length: 1000 + corrector: + enabled: true + suspend: true + schedule: "1 2 3 4 5" diff --git a/.github/workflows/dockers-dev-container-image.yml b/.github/workflows/dockers-dev-container-image.yml index 99614d4f57..06fa55b581 100644 --- a/.github/workflows/dockers-dev-container-image.yml +++ b/.github/workflows/dockers-dev-container-image.yml @@ -22,22 +22,31 @@ on: - ".github/actions/docker-build/actions.yaml" - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - - "dockers/ci/**" - "dockers/dev/**" + - "Makefile" + - "Makefile.d/**" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" pull_request: paths: - ".github/actions/docker-build/actions.yaml" - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - - "dockers/ci/**" - "dockers/dev/**" + - "Makefile" + - "Makefile.d/**" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - - "dockers/ci/**" - "dockers/dev/**" + - "Makefile" + - "Makefile.d/**" + - "versions/GO_VERSION" + - "versions/NGT_VERSION" schedule: - cron: "0 1 * * *" diff --git a/.github/workflows/dockers-index-correction.yml b/.github/workflows/dockers-index-correction.yml new file mode 100644 index 0000000000..3d6ada85a4 --- /dev/null +++ b/.github/workflows/dockers-index-correction.yml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: index-correction" +on: + push: + branches: + - main + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-index-correction.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/correction/**" + - "cmd/index/job/correction/**" + - "dockers/index/job/correction/Dockerfile" + - "versions/GO_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-index-correction.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/correction/**" + - "cmd/index/job/correction/**" + - "dockers/index/job/correction/Dockerfile" + - "versions/GO_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-index-correction.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/correction/**" + - "cmd/index/job/correction/**" + - "dockers/index/job/correction/Dockerfile" + - "versions/GO_VERSION" + +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-correction + secrets: inherit diff --git a/.github/workflows/dockers-index-creation.yml b/.github/workflows/dockers-index-creation.yml new file mode 100644 index 0000000000..1d09435f0a --- /dev/null +++ b/.github/workflows/dockers-index-creation.yml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: index-creation" +on: + push: + branches: + - main + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-index-creation.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/creation/**" + - "cmd/index/job/creation/**" + - "dockers/index/job/creation/Dockerfile" + - "versions/GO_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-index-creation.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/creation/**" + - "cmd/index/job/creation/**" + - "dockers/index/job/creation/Dockerfile" + - "versions/GO_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-index-creation.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/creation/**" + - "cmd/index/job/creation/**" + - "dockers/index/job/creation/Dockerfile" + - "versions/GO_VERSION" + +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-creation + secrets: inherit diff --git a/.github/workflows/dockers-index-save.yml b/.github/workflows/dockers-index-save.yml new file mode 100644 index 0000000000..a6570e4bdf --- /dev/null +++ b/.github/workflows/dockers-index-save.yml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: index-save" +on: + push: + branches: + - main + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/dockers-index-save.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/save/**" + - "cmd/index/job/save/**" + - "dockers/index/job/save/Dockerfile" + - "versions/GO_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-index-save.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/save/**" + - "cmd/index/job/save/**" + - "dockers/index/job/save/Dockerfile" + - "versions/GO_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/actions.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-index-save.yml" + - "go.mod" + - "go.sum" + - "internal/**" + - "!internal/**/*_test.go" + - "!internal/db/**" + - "!internal/k8s/**" + - "apis/grpc/**" + - "pkg/index/job/save/**" + - "cmd/index/job/save/**" + - "dockers/index/job/save/Dockerfile" + - "versions/GO_VERSION" + +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-save + secrets: inherit diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0f233d8f1b..1ce9ef1d7b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -227,47 +227,41 @@ jobs: env: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - # e2e-agent-and-sidecar: - # name: "E2E Agent & Sidecar test" - # needs: [dump-contexts-to-log] - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # steps: - # - uses: actions/checkout@v3 - # - # - name: Set Git config - # run: | - # git config --global --add safe.directory ${GITHUB_WORKSPACE} - # - # - name: Setup E2E environment - # id: setup_e2e - # uses: ./.github/actions/setup-e2e - # with: - # target_images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar - # - # - name: Deploy Vald - # id: deploy_vald - # uses: ./.github/actions/e2e-deploy-vald - # with: - # require_minio: "true" - # helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - # values: .github/helm/values/values-agent-sidecar.yaml - # wait_for_selector: app=vald-agent-ngt - # wait_for_timeout: 29m - # - # - name: Run E2E Agent & Sidecar - # run: | - # make hack/benchmark/assets/dataset/${{ env.DATASET }} - # make E2E_BIND_PORT=8081 \ - # E2E_DATASET_NAME=${{ env.DATASET }} \ - # E2E_INSERT_COUNT=10000 \ - # E2E_SEARCH_COUNT=4000 \ - # E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ - # E2E_TARGET_POD_NAME=${POD_NAME} \ - # E2E_TARGET_NAMESPACE=default \ - # e2e/sidecar - # env: - # POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-jobs: + name: "E2E test (Jobs)" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + + - name: Run E2E Jobs + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_INSERT_COUNT=10000\ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/index/job/correction + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-with-mirror: name: "E2E test (Stream CRUD) with mirror" @@ -334,12 +328,12 @@ jobs: name: "Slack notification" if: startsWith( github.ref, 'refs/tags/') needs: - # - e2e-agent-and-sidecar - e2e-multiapis-crud - e2e-stream-crud - e2e-stream-crud-for-operator - e2e-stream-crud-skip-exist-check - e2e-stream-crud-with-mirror + - e2e-jobs runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index d343da197b..620d1a31ea 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -18,6 +18,7 @@ on: push: branches: - main + pull_request: jobs: dump-contexts-to-log: @@ -27,7 +28,7 @@ jobs: - uses: ./.github/actions/dump-context format: - name: Run formatter + if: ${{ github.event_name == 'push' }} runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly @@ -90,3 +91,36 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: https://api.github.com/repos/vdaas/vald/pulls BRANCH_NAME: ${{ steps.switch_to_new_branch.outputs.BRANCH_NAME }} + + check-format-diff: + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false }} + runs-on: ubuntu-latest + container: + image: ghcr.io/vdaas/vald/vald-ci-container:nightly + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.DISPATCH_TOKEN }} + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Setup Go environment + uses: ./.github/actions/setup-go + + - name: Run formatter and license.go + run: | + make deps/install + make format + git checkout go.mod go.sum + + - name: Check format difference + run: | + if git diff --quiet --exit-code; then + echo "Nothing updated" + else + echo "Please execute \`make format\` locally." + exit 1 + fi diff --git a/.github/workflows/update-pull-request-and-issue-template.yml b/.github/workflows/update-pull-request-and-issue-template.yml index 6702cf7bc9..4df04e3ae6 100644 --- a/.github/workflows/update-pull-request-and-issue-template.yml +++ b/.github/workflows/update-pull-request-and-issue-template.yml @@ -55,21 +55,12 @@ jobs: git checkout main git checkout -b ${BRANCH_NAME} - GO_VERSION=$(make version/go) - NGT_VERSION=$(make version/ngt) - KUBECTL_VERSION=$(make version/k8s) + make update/template - sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - sed -i -e "s/^- Go Version: .*$/- Go Version: ${GO_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - - sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - sed -i -e "s/^- NGT Version: .*$/- NGT Version: ${NGT_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md - - sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/bug_report.md - sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/ISSUE_TEMPLATE/security_issue_report.md - sed -i -e "s/^- Kubernetes Version: .*$/- Kubernetes Version: ${KUBECTL_VERSION}/" .github/PULL_REQUEST_TEMPLATE.md + if git diff --quiet --exit-code; then + echo "Nothing updated" + exit 0 + fi git add .github/PULL_REQUEST_TEMPLATE.md .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/security_issue_report.md git commit -S --signoff -m ":robot: Automatically update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE" diff --git a/.golangci.yml b/.golangci.yml index a3bb179b38..065e2b5550 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -51,7 +51,6 @@ linters: - gochecknoinits - goconst - godot - - godox - gofumpt - goimports - gomnd @@ -99,6 +98,7 @@ linters: # - gocognit # - gocritic # - gocyclo + # - godox # - goerr113 # - gofmt # - goheader diff --git a/Makefile b/Makefile index b0b445e26f..c6d6afcc9e 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,9 @@ MIRROR_GATEWAY_IMAGE = $(NAME)-mirror-gateway HELM_OPERATOR_IMAGE = $(NAME)-helm-operator LB_GATEWAY_IMAGE = $(NAME)-lb-gateway LOADTEST_IMAGE = $(NAME)-loadtest +INDEX_CORRECTION_IMAGE = $(NAME)-index-correction +INDEX_CREATION_IMAGE = $(NAME)-index-creation +INDEX_SAVE_IMAGE = $(NAME)-index-save MANAGER_INDEX_IMAGE = $(NAME)-manager-index MAINTAINER = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>" @@ -388,6 +391,7 @@ update: \ update/libs \ proto/all \ deps \ + update/template \ format \ go/deps @@ -577,3 +581,4 @@ include Makefile.d/kind.mk include Makefile.d/proto.mk include Makefile.d/test.mk include Makefile.d/tools.mk +include Makefile.d/minikube.mk diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index b3fb6de69e..95d0b777fe 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -234,6 +234,93 @@ cmd/manager/index/index: \ $(dir $@)main.go $@ -version +cmd/index/job/correction/index-correction: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find $(ROOTDIR)/cmd/index/job/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find $(ROOTDIR)/pkg/index/job/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + $(eval CGO_ENABLED = 0) + CGO_ENABLED=$(CGO_ENABLED) \ + GO111MODULE=on \ + GOPRIVATE=$(GOPRIVATE) \ + go build \ + --ldflags "-w -extldflags=-static \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ + -buildid=" \ + -mod=readonly \ + -modcacherw \ + -a \ + -tags "osusergo netgo static_build" \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + $@ -version + +cmd/index/job/creation/index-creation: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find $(ROOTDIR)/cmd/index/job/creation -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find $(ROOTDIR)/pkg/index/job/creation -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + $(eval CGO_ENABLED = 0) + CGO_ENABLED=$(CGO_ENABLED) \ + GO111MODULE=on \ + GOPRIVATE=$(GOPRIVATE) \ + go build \ + --ldflags "-w -extldflags=-static \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ + -buildid=" \ + -mod=readonly \ + -modcacherw \ + -a \ + -tags "osusergo netgo static_build" \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + $@ -version + +cmd/index/job/save/index-save: \ + $(GO_SOURCES_INTERNAL) \ + $(PBGOS) \ + $(shell find $(ROOTDIR)/cmd/index/job/save -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \ + $(shell find $(ROOTDIR)/pkg/index/job/save -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') + $(eval CGO_ENABLED = 0) + CGO_ENABLED=$(CGO_ENABLED) \ + GO111MODULE=on \ + GOPRIVATE=$(GOPRIVATE) \ + go build \ + --ldflags "-w -extldflags=-static \ + -X '$(GOPKG)/internal/info.Version=$(VERSION)' \ + -X '$(GOPKG)/internal/info.GitCommit=$(GIT_COMMIT)' \ + -X '$(GOPKG)/internal/info.BuildTime=$(DATETIME)' \ + -X '$(GOPKG)/internal/info.GoVersion=$(GO_VERSION)' \ + -X '$(GOPKG)/internal/info.GoOS=$(GOOS)' \ + -X '$(GOPKG)/internal/info.GoArch=$(GOARCH)' \ + -X '$(GOPKG)/internal/info.CGOEnabled=$(CGO_ENABLED)' \ + -X '$(GOPKG)/internal/info.BuildCPUInfoFlags=$(CPU_INFO_FLAGS)' \ + -buildid=" \ + -mod=readonly \ + -modcacherw \ + -a \ + -tags "osusergo netgo static_build" \ + -trimpath \ + -o $@ \ + $(dir $@)main.go + $@ -version + .PHONY: binary/build/zip ## build all binaries and zip them binary/build/zip: \ diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index 4f03a42add..da430a2abd 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -24,9 +24,10 @@ update/libs: \ update/helm-docs \ update/helm-operator \ update/jaeger-operator \ + update/k3s \ update/kind \ - update/kubectl \ update/kube-linter \ + update/kubectl \ update/ngt \ update/prometheus-stack \ update/protobuf \ @@ -77,6 +78,11 @@ go/example/deps: update/chaos-mesh: curl --silent https://api.github.com/repos/chaos-mesh/chaos-mesh/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/CHAOS_MESH_VERSION +.PHONY: update/k3s +## update k3s version +update/k3s: + curl --silent https://hub.docker.com/v2/repositories/rancher/k3s/tags | jq -r '.results[].name' | grep -E '.*-k3s1$$' | sort -V | tail -n 1 > $(ROOTDIR)/versions/K3S_VERSION + .PHONY: update/go ## update go version update/go: @@ -166,3 +172,21 @@ update/vald: ## update vald client library made by clojure self version update/valdcli: curl --silent https://api.github.com/repos/vdaas/vald-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION + +.PHONY: update/template +## update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE +update/template: + $(eval GO_VERSION := $(shell $(MAKE) -s version/go)) + $(eval NGT_VERSION := $(shell $(MAKE) -s version/ngt)) + $(eval KUBECTL_VERSION := $(shell $(MAKE) -s version/k8s)) + sed -i -e "s/^- Go Version: .*$$/- Go Version: $(GO_VERSION)/" $(ROOTDIR)/.github/ISSUE_TEMPLATE/bug_report.md + sed -i -e "s/^- Go Version: .*$$/- Go Version: $(GO_VERSION)/" $(ROOTDIR)/.github/ISSUE_TEMPLATE/security_issue_report.md + sed -i -e "s/^- Go Version: .*$$/- Go Version: $(GO_VERSION)/" $(ROOTDIR)/.github/PULL_REQUEST_TEMPLATE.md + + sed -i -e "s/^- NGT Version: .*$$/- NGT Version: $(NGT_VERSION)/" $(ROOTDIR)/.github/ISSUE_TEMPLATE/bug_report.md + sed -i -e "s/^- NGT Version: .*$$/- NGT Version: $(NGT_VERSION)/" $(ROOTDIR)/.github/ISSUE_TEMPLATE/security_issue_report.md + sed -i -e "s/^- NGT Version: .*$$/- NGT Version: $(NGT_VERSION)/" $(ROOTDIR)/.github/PULL_REQUEST_TEMPLATE.md + + sed -i -e "s/^- Kubernetes Version: .*$$/- Kubernetes Version: $(KUBECTL_VERSION)/" $(ROOTDIR)/.github/ISSUE_TEMPLATE/bug_report.md + sed -i -e "s/^- Kubernetes Version: .*$$/- Kubernetes Version: $(KUBECTL_VERSION)/" $(ROOTDIR)/.github/ISSUE_TEMPLATE/security_issue_report.md + sed -i -e "s/^- Kubernetes Version: .*$$/- Kubernetes Version: $(KUBECTL_VERSION)/" $(ROOTDIR)/.github/PULL_REQUEST_TEMPLATE.md diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 99d0b32eb4..20025548f4 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -203,3 +203,45 @@ docker/build/loadtest: -t $(ORG)/$(LOADTEST_IMAGE):$(TAG) . \ --build-arg MAINTAINER=$(MAINTAINER) \ --build-arg GO_VERSION=$(GO_VERSION) + +.PHONY: docker/name/index-correction +docker/name/index-correction: + @echo "$(ORG)/$(INDEX_CORRECTION_IMAGE)" + +.PHONY: docker/build/index-correction +## build index-correction image +docker/build/index-correction: + $(DOCKER) build \ + $(DOCKER_OPTS) \ + -f dockers/index/job/correction/Dockerfile \ + -t $(ORG)/$(INDEX_CORRECTION_IMAGE):$(TAG) . \ + --build-arg MAINTAINER=$(MAINTAINER) \ + --build-arg GO_VERSION=$(GO_VERSION) + +.PHONY: docker/name/index-creation +docker/name/index-creation: + @echo "$(ORG)/$(INDEX_CREATION_IMAGE)" + +.PHONY: docker/build/index-creation +## build index-creation image +docker/build/index-creation: + $(DOCKER) build \ + $(DOCKER_OPTS) \ + -f dockers/index/job/creation/Dockerfile \ + -t $(ORG)/$(INDEX_CREATION_IMAGE):$(TAG) . \ + --build-arg MAINTAINER=$(MAINTAINER) \ + --build-arg GO_VERSION=$(GO_VERSION) + +.PHONY: docker/name/index-save +docker/name/index-save: + @echo "$(ORG)/$(INDEX_SAVE_IMAGE)" + +.PHONY: docker/build/index-save +## build index-save image +docker/build/index-save: + $(DOCKER) build \ + $(DOCKER_OPTS) \ + -f dockers/index/job/save/Dockerfile \ + -t $(ORG)/$(INDEX_SAVE_IMAGE):$(TAG) . \ + --build-arg MAINTAINER=$(MAINTAINER) \ + --build-arg GO_VERSION=$(GO_VERSION) diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index 8cd6e43800..f830343788 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -69,6 +69,11 @@ e2e/remove/timestamp: e2e/insert/search: $(call run-e2e-crud-test,-run TestE2EInsertAndSearch) +.PHONY: e2e/index/job/correction +## run index correction job e2e +e2e/index/job/correction: + $(call run-e2e-crud-test,-run TestE2EIndexJobCorrection) + .PHONY: e2e/maxdim ## run e2e/maxdim e2e/maxdim: @@ -79,4 +84,3 @@ e2e/maxdim: e2e/sidecar: $(call run-e2e-sidecar-test,-run TestE2EForSidecar) - diff --git a/Makefile.d/helm.mk b/Makefile.d/helm.mk index 2137a41de6..dd0a8ded9b 100644 --- a/Makefile.d/helm.mk +++ b/Makefile.d/helm.mk @@ -29,8 +29,8 @@ helm-docs/install: $(BINDIR)/helm-docs $(BINDIR)/helm-docs: mkdir -p $(BINDIR) cd $(TEMP_DIR) \ - && curl -LO https://github.com/norwoodj/helm-docs/releases/download/v$(HELM_DOCS_VERSION)/helm-docs_$(UNAME)_$(ARCH).tar.gz \ - && tar xzvf helm-docs_$(UNAME)_$(ARCH).tar.gz \ + && curl -LO https://github.com/norwoodj/helm-docs/releases/download/v$(HELM_DOCS_VERSION)/helm-docs_$(HELM_DOCS_VERSION)_$(UNAME)_$(ARCH).tar.gz \ + && tar xzvf helm-docs_$(HELM_DOCS_VERSION)_$(UNAME)_$(ARCH).tar.gz \ && mv helm-docs $(BINDIR)/helm-docs .PHONY: helm/package/vald diff --git a/Makefile.d/k3d.mk b/Makefile.d/k3d.mk index 7445b8e5a9..5bda1e19c8 100644 --- a/Makefile.d/k3d.mk +++ b/Makefile.d/k3d.mk @@ -34,8 +34,6 @@ k3d/start: # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) --agents $(K3D_NODES) -v "/lib/modules:/lib/modules" --host-pid-mode=true # $(K3D_COMMAND) cluster create $(K3D_CLUSTER_NAME) -p "8081:80@loadbalancer" --agents $(K3D_NODES) --k3s-arg '--disable=traefik@all' export KUBECONFIG="$(shell sudo $(K3D_COMMAND) kubeconfig merge -o $(TEMP_DIR)/k3d_$(K3D_CLUSTER_NAME)_kubeconfig.yaml $(K3D_CLUSTER_NAME))" - kubectl apply -f https://projectcontour.io/quickstart/operator.yaml - kubectl apply -f https://projectcontour.io/quickstart/contour-custom-resource.yaml .PHONY: k3d/restart ## restart k3d (kubernetes in docker) cluster diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index eb4f62d56d..05d58611e9 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -27,22 +27,26 @@ k8s/manifest/clean: k8s/agent \ k8s/discoverer \ k8s/gateway \ - k8s/manager + k8s/manager \ + k8s/index .PHONY: k8s/manifest/update ## update k8s manifests using helm templates k8s/manifest/update: \ k8s/manifest/clean helm template \ - --values $(HELM_VALUES) \ - --output-dir $(TEMP_DIR) \ - charts/vald + --values $(HELM_VALUES) \ + $(HELM_EXTRA_OPTIONS) \ + --output-dir $(TEMP_DIR) \ + charts/vald mkdir -p k8s/gateway mkdir -p k8s/manager + mkdir -p k8s/index/job mv $(TEMP_DIR)/vald/templates/agent k8s/agent mv $(TEMP_DIR)/vald/templates/discoverer k8s/discoverer mv $(TEMP_DIR)/vald/templates/gateway/lb k8s/gateway/lb mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index + mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction rm -rf $(TEMP_DIR) .PHONY: k8s/manifest/helm-operator/clean @@ -88,6 +92,7 @@ k8s/vald/deploy: kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/lb || true kubectl apply -f $(TEMP_DIR)/vald/crds || true kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/mirror || true + kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/correction || true rm -rf $(TEMP_DIR) kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n" @@ -108,6 +113,7 @@ k8s/vald/delete: --output-dir $(TEMP_DIR) \ charts/vald kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/mirror + kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index kubectl delete -f $(TEMP_DIR)/vald/templates/discoverer diff --git a/Makefile.d/minikube.mk b/Makefile.d/minikube.mk new file mode 100644 index 0000000000..7ba03e8440 --- /dev/null +++ b/Makefile.d/minikube.mk @@ -0,0 +1,42 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +.PHONY: minikube/install +minikube/install: $(BINDIR)/minikube + +$(BINDIR)/minikube: + mkdir -p $(BINDIR) + curl -L https://storage.googleapis.com/minikube/releases/latest/minikube-$(shell echo $(UNAME) | tr '[:upper:]' '[:lower:]')-$(subst x86_64,amd64,$(shell echo $(ARCH) | tr '[:upper:]' '[:lower:]')) -o $(BINDIR)/minikube + chmod a+x $(BINDIR)/minikube + +# Start minikube with CSI Driver and Volume Snapshots support +# Only use this for development related to Volume Snapshots. Usually k3d is faster. +.PHONY: minikube/start +minikube/start: + minikube start --force + minikube addons enable volumesnapshots + minikube addons enable csi-hostpath-driver + minikube addons disable storage-provisioner + minikube addons disable default-storageclass + kubectl patch storageclass csi-hostpath-sc -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' + +.PHONY: minikube/delete +minikube/delete: + minikube delete + +.PHONY: minikube/restart +minikube/restart: + @make minikube/delete + @make minikube/start diff --git a/Makefile.d/proto.mk b/Makefile.d/proto.mk index a30940bbf1..2462f8e646 100644 --- a/Makefile.d/proto.mk +++ b/Makefile.d/proto.mk @@ -58,8 +58,7 @@ proto/deps: \ $(GOPATH)/src/github.com/planetscale/vtprotobuf \ $(GOPATH)/src/github.com/protocolbuffers/protobuf \ $(GOPATH)/src/google.golang.org/genproto \ - $(GOPATH)/src/google.golang.org/protobuf \ - $(ROOTDIR)/apis/proto/v1/rpc/error_details.proto + $(GOPATH)/src/google.golang.org/protobuf .PHONY: proto/clean/deps ## uninstall all protobuf dependencies @@ -79,8 +78,7 @@ proto/clean/deps: $(GOPATH)/src/github.com/planetscale/vtprotobuf \ $(GOPATH)/src/github.com/protocolbuffers/protobuf \ $(GOPATH)/src/google.golang.org/genproto \ - $(GOPATH)/src/google.golang.org/protobuf \ - $(ROOTDIR)/apis/proto/v1/rpc/error_details.proto + $(GOPATH)/src/google.golang.org/protobuf $(GOPATH)/src/github.com/protocolbuffers/protobuf: diff --git a/apis/grpc/v1/agent/core/agent.pb.go b/apis/grpc/v1/agent/core/agent.pb.go index 9774d6fc08..5bbab84ca0 100644 --- a/apis/grpc/v1/agent/core/agent.pb.go +++ b/apis/grpc/v1/agent/core/agent.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/agent/core/agent.proto package core diff --git a/apis/grpc/v1/agent/sidecar/sidecar.pb.go b/apis/grpc/v1/agent/sidecar/sidecar.pb.go index 1be4b3adee..ca6b2cba93 100644 --- a/apis/grpc/v1/agent/sidecar/sidecar.pb.go +++ b/apis/grpc/v1/agent/sidecar/sidecar.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/agent/sidecar/sidecar.proto package sidecar diff --git a/apis/grpc/v1/discoverer/discoverer.pb.go b/apis/grpc/v1/discoverer/discoverer.pb.go index 5e39492195..6afe527031 100644 --- a/apis/grpc/v1/discoverer/discoverer.pb.go +++ b/apis/grpc/v1/discoverer/discoverer.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/discoverer/discoverer.proto package discoverer diff --git a/apis/grpc/v1/filter/egress/egress_filter.pb.go b/apis/grpc/v1/filter/egress/egress_filter.pb.go index 79ac61a5c7..0bbfce5f77 100644 --- a/apis/grpc/v1/filter/egress/egress_filter.pb.go +++ b/apis/grpc/v1/filter/egress/egress_filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/filter/egress/egress_filter.proto package egress diff --git a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go index 1a94009065..9cbaa9af3b 100644 --- a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go +++ b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/filter/ingress/ingress_filter.proto package ingress diff --git a/apis/grpc/v1/manager/index/index_manager.pb.go b/apis/grpc/v1/manager/index/index_manager.pb.go index 785a6d43b4..7c03f50043 100644 --- a/apis/grpc/v1/manager/index/index_manager.pb.go +++ b/apis/grpc/v1/manager/index/index_manager.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/manager/index/index_manager.proto package index diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go index 53e0060064..0f595076ff 100644 --- a/apis/grpc/v1/payload/payload.pb.go +++ b/apis/grpc/v1/payload/payload.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/payload/payload.proto package payload @@ -545,45 +545,6 @@ func (*Info) Descriptor() ([]byte, []int) { return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9} } -// Mirror related messages. -type Mirror struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *Mirror) Reset() { - *x = Mirror{} - if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Mirror) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Mirror) ProtoMessage() {} - -func (x *Mirror) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Mirror.ProtoReflect.Descriptor instead. -func (*Mirror) Descriptor() ([]byte, []int) { - return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{10} -} - // Represent an empty message. type Empty struct { state protoimpl.MessageState @@ -594,7 +555,7 @@ type Empty struct { func (x *Empty) Reset() { *x = Empty{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[11] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -607,7 +568,7 @@ func (x *Empty) String() string { func (*Empty) ProtoMessage() {} func (x *Empty) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[11] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -620,7 +581,7 @@ func (x *Empty) ProtoReflect() protoreflect.Message { // Deprecated: Use Empty.ProtoReflect.Descriptor instead. func (*Empty) Descriptor() ([]byte, []int) { - return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{11} + return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{10} } // Represent a search request. @@ -638,7 +599,7 @@ type Search_Request struct { func (x *Search_Request) Reset() { *x = Search_Request{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[12] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -651,7 +612,7 @@ func (x *Search_Request) String() string { func (*Search_Request) ProtoMessage() {} func (x *Search_Request) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[12] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -694,7 +655,7 @@ type Search_MultiRequest struct { func (x *Search_MultiRequest) Reset() { *x = Search_MultiRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[13] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -707,7 +668,7 @@ func (x *Search_MultiRequest) String() string { func (*Search_MultiRequest) ProtoMessage() {} func (x *Search_MultiRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[13] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -745,7 +706,7 @@ type Search_IDRequest struct { func (x *Search_IDRequest) Reset() { *x = Search_IDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[14] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -758,7 +719,7 @@ func (x *Search_IDRequest) String() string { func (*Search_IDRequest) ProtoMessage() {} func (x *Search_IDRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[14] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -801,7 +762,7 @@ type Search_MultiIDRequest struct { func (x *Search_MultiIDRequest) Reset() { *x = Search_MultiIDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[15] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -814,7 +775,7 @@ func (x *Search_MultiIDRequest) String() string { func (*Search_MultiIDRequest) ProtoMessage() {} func (x *Search_MultiIDRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[15] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -854,7 +815,7 @@ type Search_ObjectRequest struct { func (x *Search_ObjectRequest) Reset() { *x = Search_ObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[16] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -867,7 +828,7 @@ func (x *Search_ObjectRequest) String() string { func (*Search_ObjectRequest) ProtoMessage() {} func (x *Search_ObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[16] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -917,7 +878,7 @@ type Search_MultiObjectRequest struct { func (x *Search_MultiObjectRequest) Reset() { *x = Search_MultiObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[17] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -930,7 +891,7 @@ func (x *Search_MultiObjectRequest) String() string { func (*Search_MultiObjectRequest) ProtoMessage() {} func (x *Search_MultiObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[17] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -982,7 +943,7 @@ type Search_Config struct { func (x *Search_Config) Reset() { *x = Search_Config{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[18] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -995,7 +956,7 @@ func (x *Search_Config) String() string { func (*Search_Config) ProtoMessage() {} func (x *Search_Config) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[18] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1089,7 +1050,7 @@ type Search_Response struct { func (x *Search_Response) Reset() { *x = Search_Response{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[19] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1102,7 +1063,7 @@ func (x *Search_Response) String() string { func (*Search_Response) ProtoMessage() {} func (x *Search_Response) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[19] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1145,7 +1106,7 @@ type Search_Responses struct { func (x *Search_Responses) Reset() { *x = Search_Responses{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[20] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1158,7 +1119,7 @@ func (x *Search_Responses) String() string { func (*Search_Responses) ProtoMessage() {} func (x *Search_Responses) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[20] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1197,7 +1158,7 @@ type Search_StreamResponse struct { func (x *Search_StreamResponse) Reset() { *x = Search_StreamResponse{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[21] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1210,7 +1171,7 @@ func (x *Search_StreamResponse) String() string { func (*Search_StreamResponse) ProtoMessage() {} func (x *Search_StreamResponse) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[21] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1280,7 +1241,7 @@ type Filter_Target struct { func (x *Filter_Target) Reset() { *x = Filter_Target{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[22] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1293,7 +1254,7 @@ func (x *Filter_Target) String() string { func (*Filter_Target) ProtoMessage() {} func (x *Filter_Target) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[22] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1336,7 +1297,7 @@ type Filter_Config struct { func (x *Filter_Config) Reset() { *x = Filter_Config{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[23] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1349,7 +1310,7 @@ func (x *Filter_Config) String() string { func (*Filter_Config) ProtoMessage() {} func (x *Filter_Config) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[23] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1387,7 +1348,7 @@ type Insert_Request struct { func (x *Insert_Request) Reset() { *x = Insert_Request{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[24] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1400,7 +1361,7 @@ func (x *Insert_Request) String() string { func (*Insert_Request) ProtoMessage() {} func (x *Insert_Request) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[24] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1443,7 +1404,7 @@ type Insert_MultiRequest struct { func (x *Insert_MultiRequest) Reset() { *x = Insert_MultiRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[25] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1456,7 +1417,7 @@ func (x *Insert_MultiRequest) String() string { func (*Insert_MultiRequest) ProtoMessage() {} func (x *Insert_MultiRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[25] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1496,7 +1457,7 @@ type Insert_ObjectRequest struct { func (x *Insert_ObjectRequest) Reset() { *x = Insert_ObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[26] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1509,7 +1470,7 @@ func (x *Insert_ObjectRequest) String() string { func (*Insert_ObjectRequest) ProtoMessage() {} func (x *Insert_ObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[26] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1559,7 +1520,7 @@ type Insert_MultiObjectRequest struct { func (x *Insert_MultiObjectRequest) Reset() { *x = Insert_MultiObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[27] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1572,7 +1533,7 @@ func (x *Insert_MultiObjectRequest) String() string { func (*Insert_MultiObjectRequest) ProtoMessage() {} func (x *Insert_MultiObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[27] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1612,7 +1573,7 @@ type Insert_Config struct { func (x *Insert_Config) Reset() { *x = Insert_Config{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[28] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1625,7 +1586,7 @@ func (x *Insert_Config) String() string { func (*Insert_Config) ProtoMessage() {} func (x *Insert_Config) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[28] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1677,7 +1638,7 @@ type Update_Request struct { func (x *Update_Request) Reset() { *x = Update_Request{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[29] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1690,7 +1651,7 @@ func (x *Update_Request) String() string { func (*Update_Request) ProtoMessage() {} func (x *Update_Request) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[29] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1733,7 +1694,7 @@ type Update_MultiRequest struct { func (x *Update_MultiRequest) Reset() { *x = Update_MultiRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[30] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1746,7 +1707,7 @@ func (x *Update_MultiRequest) String() string { func (*Update_MultiRequest) ProtoMessage() {} func (x *Update_MultiRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[30] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1786,7 +1747,7 @@ type Update_ObjectRequest struct { func (x *Update_ObjectRequest) Reset() { *x = Update_ObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[31] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1799,7 +1760,7 @@ func (x *Update_ObjectRequest) String() string { func (*Update_ObjectRequest) ProtoMessage() {} func (x *Update_ObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[31] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1849,7 +1810,7 @@ type Update_MultiObjectRequest struct { func (x *Update_MultiObjectRequest) Reset() { *x = Update_MultiObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[32] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1862,7 +1823,7 @@ func (x *Update_MultiObjectRequest) String() string { func (*Update_MultiObjectRequest) ProtoMessage() {} func (x *Update_MultiObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[32] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1905,7 +1866,7 @@ type Update_Config struct { func (x *Update_Config) Reset() { *x = Update_Config{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[33] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1918,7 +1879,7 @@ func (x *Update_Config) String() string { func (*Update_Config) ProtoMessage() {} func (x *Update_Config) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[33] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1977,7 +1938,7 @@ type Upsert_Request struct { func (x *Upsert_Request) Reset() { *x = Upsert_Request{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[34] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1990,7 +1951,7 @@ func (x *Upsert_Request) String() string { func (*Upsert_Request) ProtoMessage() {} func (x *Upsert_Request) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[34] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2033,7 +1994,7 @@ type Upsert_MultiRequest struct { func (x *Upsert_MultiRequest) Reset() { *x = Upsert_MultiRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[35] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2046,7 +2007,7 @@ func (x *Upsert_MultiRequest) String() string { func (*Upsert_MultiRequest) ProtoMessage() {} func (x *Upsert_MultiRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[35] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2086,7 +2047,7 @@ type Upsert_ObjectRequest struct { func (x *Upsert_ObjectRequest) Reset() { *x = Upsert_ObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[36] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2099,7 +2060,7 @@ func (x *Upsert_ObjectRequest) String() string { func (*Upsert_ObjectRequest) ProtoMessage() {} func (x *Upsert_ObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[36] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2149,7 +2110,7 @@ type Upsert_MultiObjectRequest struct { func (x *Upsert_MultiObjectRequest) Reset() { *x = Upsert_MultiObjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[37] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2162,7 +2123,7 @@ func (x *Upsert_MultiObjectRequest) String() string { func (*Upsert_MultiObjectRequest) ProtoMessage() {} func (x *Upsert_MultiObjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[37] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2205,7 +2166,7 @@ type Upsert_Config struct { func (x *Upsert_Config) Reset() { *x = Upsert_Config{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[38] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2218,7 +2179,7 @@ func (x *Upsert_Config) String() string { func (*Upsert_Config) ProtoMessage() {} func (x *Upsert_Config) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[38] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2277,7 +2238,7 @@ type Remove_Request struct { func (x *Remove_Request) Reset() { *x = Remove_Request{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[39] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2290,7 +2251,7 @@ func (x *Remove_Request) String() string { func (*Remove_Request) ProtoMessage() {} func (x *Remove_Request) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[39] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2333,7 +2294,7 @@ type Remove_MultiRequest struct { func (x *Remove_MultiRequest) Reset() { *x = Remove_MultiRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[40] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2346,7 +2307,7 @@ func (x *Remove_MultiRequest) String() string { func (*Remove_MultiRequest) ProtoMessage() {} func (x *Remove_MultiRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[40] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2382,7 +2343,7 @@ type Remove_TimestampRequest struct { func (x *Remove_TimestampRequest) Reset() { *x = Remove_TimestampRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[41] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2395,7 +2356,7 @@ func (x *Remove_TimestampRequest) String() string { func (*Remove_TimestampRequest) ProtoMessage() {} func (x *Remove_TimestampRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[41] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2432,7 +2393,7 @@ type Remove_Timestamp struct { func (x *Remove_Timestamp) Reset() { *x = Remove_Timestamp{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[42] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2445,7 +2406,7 @@ func (x *Remove_Timestamp) String() string { func (*Remove_Timestamp) ProtoMessage() {} func (x *Remove_Timestamp) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[42] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2490,7 +2451,7 @@ type Remove_Config struct { func (x *Remove_Config) Reset() { *x = Remove_Config{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[43] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2503,7 +2464,7 @@ func (x *Remove_Config) String() string { func (*Remove_Config) ProtoMessage() {} func (x *Remove_Config) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[43] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2548,7 +2509,7 @@ type Object_VectorRequest struct { func (x *Object_VectorRequest) Reset() { *x = Object_VectorRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[44] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2561,7 +2522,7 @@ func (x *Object_VectorRequest) String() string { func (*Object_VectorRequest) ProtoMessage() {} func (x *Object_VectorRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[44] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2606,7 +2567,7 @@ type Object_Distance struct { func (x *Object_Distance) Reset() { *x = Object_Distance{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[45] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2619,7 +2580,7 @@ func (x *Object_Distance) String() string { func (*Object_Distance) ProtoMessage() {} func (x *Object_Distance) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[45] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2665,7 +2626,7 @@ type Object_StreamDistance struct { func (x *Object_StreamDistance) Reset() { *x = Object_StreamDistance{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[46] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2678,7 +2639,7 @@ func (x *Object_StreamDistance) String() string { func (*Object_StreamDistance) ProtoMessage() {} func (x *Object_StreamDistance) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[46] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2745,7 +2706,7 @@ type Object_ID struct { func (x *Object_ID) Reset() { *x = Object_ID{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[47] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2758,7 +2719,7 @@ func (x *Object_ID) String() string { func (*Object_ID) ProtoMessage() {} func (x *Object_ID) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[47] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2793,7 +2754,7 @@ type Object_IDs struct { func (x *Object_IDs) Reset() { *x = Object_IDs{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[48] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2806,7 +2767,7 @@ func (x *Object_IDs) String() string { func (*Object_IDs) ProtoMessage() {} func (x *Object_IDs) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[48] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2846,7 +2807,7 @@ type Object_Vector struct { func (x *Object_Vector) Reset() { *x = Object_Vector{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[49] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2859,7 +2820,7 @@ func (x *Object_Vector) String() string { func (*Object_Vector) ProtoMessage() {} func (x *Object_Vector) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[49] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2908,7 +2869,7 @@ type Object_Vectors struct { func (x *Object_Vectors) Reset() { *x = Object_Vectors{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[50] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2921,7 +2882,7 @@ func (x *Object_Vectors) String() string { func (*Object_Vectors) ProtoMessage() {} func (x *Object_Vectors) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[50] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2960,7 +2921,7 @@ type Object_StreamVector struct { func (x *Object_StreamVector) Reset() { *x = Object_StreamVector{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[51] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2973,7 +2934,7 @@ func (x *Object_StreamVector) String() string { func (*Object_StreamVector) ProtoMessage() {} func (x *Object_StreamVector) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[51] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3043,7 +3004,7 @@ type Object_ReshapeVector struct { func (x *Object_ReshapeVector) Reset() { *x = Object_ReshapeVector{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[52] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3056,7 +3017,7 @@ func (x *Object_ReshapeVector) String() string { func (*Object_ReshapeVector) ProtoMessage() {} func (x *Object_ReshapeVector) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[52] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3101,7 +3062,7 @@ type Object_Blob struct { func (x *Object_Blob) Reset() { *x = Object_Blob{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[53] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3114,7 +3075,7 @@ func (x *Object_Blob) String() string { func (*Object_Blob) ProtoMessage() {} func (x *Object_Blob) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[53] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3160,7 +3121,7 @@ type Object_StreamBlob struct { func (x *Object_StreamBlob) Reset() { *x = Object_StreamBlob{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[54] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3173,7 +3134,7 @@ func (x *Object_StreamBlob) String() string { func (*Object_StreamBlob) ProtoMessage() {} func (x *Object_StreamBlob) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[54] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3245,7 +3206,7 @@ type Object_Location struct { func (x *Object_Location) Reset() { *x = Object_Location{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[55] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3258,7 +3219,7 @@ func (x *Object_Location) String() string { func (*Object_Location) ProtoMessage() {} func (x *Object_Location) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[55] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3311,7 +3272,7 @@ type Object_StreamLocation struct { func (x *Object_StreamLocation) Reset() { *x = Object_StreamLocation{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[56] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3324,7 +3285,7 @@ func (x *Object_StreamLocation) String() string { func (*Object_StreamLocation) ProtoMessage() {} func (x *Object_StreamLocation) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[56] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3391,7 +3352,7 @@ type Object_Locations struct { func (x *Object_Locations) Reset() { *x = Object_Locations{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[57] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3404,7 +3365,7 @@ func (x *Object_Locations) String() string { func (*Object_Locations) ProtoMessage() {} func (x *Object_Locations) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[57] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3437,7 +3398,7 @@ type Object_List struct { func (x *Object_List) Reset() { *x = Object_List{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[58] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3450,7 +3411,7 @@ func (x *Object_List) String() string { func (*Object_List) ProtoMessage() {} func (x *Object_List) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[58] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3475,7 +3436,7 @@ type Object_List_Request struct { func (x *Object_List_Request) Reset() { *x = Object_List_Request{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[59] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3488,7 +3449,7 @@ func (x *Object_List_Request) String() string { func (*Object_List_Request) ProtoMessage() {} func (x *Object_List_Request) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[59] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3519,7 +3480,7 @@ type Object_List_Response struct { func (x *Object_List_Response) Reset() { *x = Object_List_Response{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[60] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3532,7 +3493,7 @@ func (x *Object_List_Response) String() string { func (*Object_List_Response) ProtoMessage() {} func (x *Object_List_Response) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[60] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3600,7 +3561,7 @@ type Control_CreateIndexRequest struct { func (x *Control_CreateIndexRequest) Reset() { *x = Control_CreateIndexRequest{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[61] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3613,7 +3574,7 @@ func (x *Control_CreateIndexRequest) String() string { func (*Control_CreateIndexRequest) ProtoMessage() {} func (x *Control_CreateIndexRequest) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[61] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3653,7 +3614,7 @@ type Discoverer_Request struct { func (x *Discoverer_Request) Reset() { *x = Discoverer_Request{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[62] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3666,7 +3627,7 @@ func (x *Discoverer_Request) String() string { func (*Discoverer_Request) ProtoMessage() {} func (x *Discoverer_Request) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[62] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3713,7 +3674,7 @@ type Info_Index struct { func (x *Info_Index) Reset() { *x = Info_Index{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[63] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3726,7 +3687,7 @@ func (x *Info_Index) String() string { func (*Info_Index) ProtoMessage() {} func (x *Info_Index) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[63] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3767,7 +3728,7 @@ type Info_Pod struct { func (x *Info_Pod) Reset() { *x = Info_Pod{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[64] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3780,7 +3741,7 @@ func (x *Info_Pod) String() string { func (*Info_Pod) ProtoMessage() {} func (x *Info_Pod) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[64] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3868,7 +3829,7 @@ type Info_Node struct { func (x *Info_Node) Reset() { *x = Info_Node{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[65] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3881,7 +3842,7 @@ func (x *Info_Node) String() string { func (*Info_Node) ProtoMessage() {} func (x *Info_Node) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[65] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3956,7 +3917,7 @@ type Info_CPU struct { func (x *Info_CPU) Reset() { *x = Info_CPU{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3969,7 +3930,7 @@ func (x *Info_CPU) String() string { func (*Info_CPU) ProtoMessage() {} func (x *Info_CPU) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4023,7 +3984,7 @@ type Info_Memory struct { func (x *Info_Memory) Reset() { *x = Info_Memory{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4036,7 +3997,7 @@ func (x *Info_Memory) String() string { func (*Info_Memory) ProtoMessage() {} func (x *Info_Memory) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4086,7 +4047,7 @@ type Info_Pods struct { func (x *Info_Pods) Reset() { *x = Info_Pods{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[68] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4099,7 +4060,7 @@ func (x *Info_Pods) String() string { func (*Info_Pods) ProtoMessage() {} func (x *Info_Pods) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[68] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4135,7 +4096,7 @@ type Info_Nodes struct { func (x *Info_Nodes) Reset() { *x = Info_Nodes{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[69] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4148,7 +4109,7 @@ func (x *Info_Nodes) String() string { func (*Info_Nodes) ProtoMessage() {} func (x *Info_Nodes) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[69] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4183,7 +4144,7 @@ type Info_IPs struct { func (x *Info_IPs) Reset() { *x = Info_IPs{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[70] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4196,7 +4157,7 @@ func (x *Info_IPs) String() string { func (*Info_IPs) ProtoMessage() {} func (x *Info_IPs) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[70] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4238,7 +4199,7 @@ type Info_Index_Count struct { func (x *Info_Index_Count) Reset() { *x = Info_Index_Count{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4251,7 +4212,7 @@ func (x *Info_Index_Count) String() string { func (*Info_Index_Count) ProtoMessage() {} func (x *Info_Index_Count) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4305,7 +4266,7 @@ type Info_Index_UUID struct { func (x *Info_Index_UUID) Reset() { *x = Info_Index_UUID{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4318,7 +4279,7 @@ func (x *Info_Index_UUID) String() string { func (*Info_Index_UUID) ProtoMessage() {} func (x *Info_Index_UUID) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4346,7 +4307,7 @@ type Info_Index_UUID_Committed struct { func (x *Info_Index_UUID_Committed) Reset() { *x = Info_Index_UUID_Committed{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[73] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4359,7 +4320,7 @@ func (x *Info_Index_UUID_Committed) String() string { func (*Info_Index_UUID_Committed) ProtoMessage() {} func (x *Info_Index_UUID_Committed) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[73] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4394,7 +4355,7 @@ type Info_Index_UUID_Uncommitted struct { func (x *Info_Index_UUID_Uncommitted) Reset() { *x = Info_Index_UUID_Uncommitted{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[74] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4407,7 +4368,7 @@ func (x *Info_Index_UUID_Uncommitted) String() string { func (*Info_Index_UUID_Uncommitted) ProtoMessage() {} func (x *Info_Index_UUID_Uncommitted) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[74] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4430,113 +4391,6 @@ func (x *Info_Index_UUID_Uncommitted) GetUuid() string { return "" } -// Represent server information. -type Mirror_Target struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The target hostname. - Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` - // The target port. - Port uint32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` -} - -func (x *Mirror_Target) Reset() { - *x = Mirror_Target{} - if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[75] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Mirror_Target) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Mirror_Target) ProtoMessage() {} - -func (x *Mirror_Target) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[75] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Mirror_Target.ProtoReflect.Descriptor instead. -func (*Mirror_Target) Descriptor() ([]byte, []int) { - return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{10, 0} -} - -func (x *Mirror_Target) GetHost() string { - if x != nil { - return x.Host - } - return "" -} - -func (x *Mirror_Target) GetPort() uint32 { - if x != nil { - return x.Port - } - return 0 -} - -// Represent the multiple Target message. -type Mirror_Targets struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The multiple target information. - Targets []*Mirror_Target `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty"` -} - -func (x *Mirror_Targets) Reset() { - *x = Mirror_Targets{} - if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[76] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Mirror_Targets) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Mirror_Targets) ProtoMessage() {} - -func (x *Mirror_Targets) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[76] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Mirror_Targets.ProtoReflect.Descriptor instead. -func (*Mirror_Targets) Descriptor() ([]byte, []int) { - return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{10, 1} -} - -func (x *Mirror_Targets) GetTargets() []*Mirror_Target { - if x != nil { - return x.Targets - } - return nil -} - var File_apis_proto_v1_payload_payload_proto protoreflect.FileDescriptor var file_apis_proto_v1_payload_payload_proto_rawDesc = []byte{ @@ -4963,22 +4817,14 @@ var file_apis_proto_v1_payload_payload_proto_rawDesc = []byte{ 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x92, 0x01, 0x02, 0x08, 0x01, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x1a, 0x15, 0x0a, 0x03, 0x49, 0x50, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x22, 0x7a, 0x0a, 0x06, 0x4d, 0x69, 0x72, 0x72, 0x6f, - 0x72, 0x1a, 0x30, 0x0a, 0x06, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, - 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, - 0x6f, 0x72, 0x74, 0x1a, 0x3e, 0x0a, 0x07, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x33, - 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x19, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x72, - 0x72, 0x6f, 0x72, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x73, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x64, 0x0a, 0x1d, - 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x0b, 0x56, - 0x61, 0x6c, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x01, 0x5a, 0x2a, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, - 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, - 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0xa2, 0x02, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x03, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x42, 0x64, 0x0a, 0x1d, 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, + 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x42, 0x0b, 0x56, 0x61, 0x6c, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x01, + 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, + 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, + 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0xa2, 0x02, 0x07, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4994,7 +4840,7 @@ func file_apis_proto_v1_payload_payload_proto_rawDescGZIP() []byte { } var file_apis_proto_v1_payload_payload_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_apis_proto_v1_payload_payload_proto_msgTypes = make([]protoimpl.MessageInfo, 77) +var file_apis_proto_v1_payload_payload_proto_msgTypes = make([]protoimpl.MessageInfo, 74) var file_apis_proto_v1_payload_payload_proto_goTypes = []interface{}{ (Search_AggregationAlgorithm)(0), // 0: payload.v1.Search.AggregationAlgorithm (Remove_Timestamp_Operator)(0), // 1: payload.v1.Remove.Timestamp.Operator @@ -5008,148 +4854,144 @@ var file_apis_proto_v1_payload_payload_proto_goTypes = []interface{}{ (*Control)(nil), // 9: payload.v1.Control (*Discoverer)(nil), // 10: payload.v1.Discoverer (*Info)(nil), // 11: payload.v1.Info - (*Mirror)(nil), // 12: payload.v1.Mirror - (*Empty)(nil), // 13: payload.v1.Empty - (*Search_Request)(nil), // 14: payload.v1.Search.Request - (*Search_MultiRequest)(nil), // 15: payload.v1.Search.MultiRequest - (*Search_IDRequest)(nil), // 16: payload.v1.Search.IDRequest - (*Search_MultiIDRequest)(nil), // 17: payload.v1.Search.MultiIDRequest - (*Search_ObjectRequest)(nil), // 18: payload.v1.Search.ObjectRequest - (*Search_MultiObjectRequest)(nil), // 19: payload.v1.Search.MultiObjectRequest - (*Search_Config)(nil), // 20: payload.v1.Search.Config - (*Search_Response)(nil), // 21: payload.v1.Search.Response - (*Search_Responses)(nil), // 22: payload.v1.Search.Responses - (*Search_StreamResponse)(nil), // 23: payload.v1.Search.StreamResponse - (*Filter_Target)(nil), // 24: payload.v1.Filter.Target - (*Filter_Config)(nil), // 25: payload.v1.Filter.Config - (*Insert_Request)(nil), // 26: payload.v1.Insert.Request - (*Insert_MultiRequest)(nil), // 27: payload.v1.Insert.MultiRequest - (*Insert_ObjectRequest)(nil), // 28: payload.v1.Insert.ObjectRequest - (*Insert_MultiObjectRequest)(nil), // 29: payload.v1.Insert.MultiObjectRequest - (*Insert_Config)(nil), // 30: payload.v1.Insert.Config - (*Update_Request)(nil), // 31: payload.v1.Update.Request - (*Update_MultiRequest)(nil), // 32: payload.v1.Update.MultiRequest - (*Update_ObjectRequest)(nil), // 33: payload.v1.Update.ObjectRequest - (*Update_MultiObjectRequest)(nil), // 34: payload.v1.Update.MultiObjectRequest - (*Update_Config)(nil), // 35: payload.v1.Update.Config - (*Upsert_Request)(nil), // 36: payload.v1.Upsert.Request - (*Upsert_MultiRequest)(nil), // 37: payload.v1.Upsert.MultiRequest - (*Upsert_ObjectRequest)(nil), // 38: payload.v1.Upsert.ObjectRequest - (*Upsert_MultiObjectRequest)(nil), // 39: payload.v1.Upsert.MultiObjectRequest - (*Upsert_Config)(nil), // 40: payload.v1.Upsert.Config - (*Remove_Request)(nil), // 41: payload.v1.Remove.Request - (*Remove_MultiRequest)(nil), // 42: payload.v1.Remove.MultiRequest - (*Remove_TimestampRequest)(nil), // 43: payload.v1.Remove.TimestampRequest - (*Remove_Timestamp)(nil), // 44: payload.v1.Remove.Timestamp - (*Remove_Config)(nil), // 45: payload.v1.Remove.Config - (*Object_VectorRequest)(nil), // 46: payload.v1.Object.VectorRequest - (*Object_Distance)(nil), // 47: payload.v1.Object.Distance - (*Object_StreamDistance)(nil), // 48: payload.v1.Object.StreamDistance - (*Object_ID)(nil), // 49: payload.v1.Object.ID - (*Object_IDs)(nil), // 50: payload.v1.Object.IDs - (*Object_Vector)(nil), // 51: payload.v1.Object.Vector - (*Object_Vectors)(nil), // 52: payload.v1.Object.Vectors - (*Object_StreamVector)(nil), // 53: payload.v1.Object.StreamVector - (*Object_ReshapeVector)(nil), // 54: payload.v1.Object.ReshapeVector - (*Object_Blob)(nil), // 55: payload.v1.Object.Blob - (*Object_StreamBlob)(nil), // 56: payload.v1.Object.StreamBlob - (*Object_Location)(nil), // 57: payload.v1.Object.Location - (*Object_StreamLocation)(nil), // 58: payload.v1.Object.StreamLocation - (*Object_Locations)(nil), // 59: payload.v1.Object.Locations - (*Object_List)(nil), // 60: payload.v1.Object.List - (*Object_List_Request)(nil), // 61: payload.v1.Object.List.Request - (*Object_List_Response)(nil), // 62: payload.v1.Object.List.Response - (*Control_CreateIndexRequest)(nil), // 63: payload.v1.Control.CreateIndexRequest - (*Discoverer_Request)(nil), // 64: payload.v1.Discoverer.Request - (*Info_Index)(nil), // 65: payload.v1.Info.Index - (*Info_Pod)(nil), // 66: payload.v1.Info.Pod - (*Info_Node)(nil), // 67: payload.v1.Info.Node - (*Info_CPU)(nil), // 68: payload.v1.Info.CPU - (*Info_Memory)(nil), // 69: payload.v1.Info.Memory - (*Info_Pods)(nil), // 70: payload.v1.Info.Pods - (*Info_Nodes)(nil), // 71: payload.v1.Info.Nodes - (*Info_IPs)(nil), // 72: payload.v1.Info.IPs - (*Info_Index_Count)(nil), // 73: payload.v1.Info.Index.Count - (*Info_Index_UUID)(nil), // 74: payload.v1.Info.Index.UUID - (*Info_Index_UUID_Committed)(nil), // 75: payload.v1.Info.Index.UUID.Committed - (*Info_Index_UUID_Uncommitted)(nil), // 76: payload.v1.Info.Index.UUID.Uncommitted - (*Mirror_Target)(nil), // 77: payload.v1.Mirror.Target - (*Mirror_Targets)(nil), // 78: payload.v1.Mirror.Targets - (*status.Status)(nil), // 79: google.rpc.Status + (*Empty)(nil), // 12: payload.v1.Empty + (*Search_Request)(nil), // 13: payload.v1.Search.Request + (*Search_MultiRequest)(nil), // 14: payload.v1.Search.MultiRequest + (*Search_IDRequest)(nil), // 15: payload.v1.Search.IDRequest + (*Search_MultiIDRequest)(nil), // 16: payload.v1.Search.MultiIDRequest + (*Search_ObjectRequest)(nil), // 17: payload.v1.Search.ObjectRequest + (*Search_MultiObjectRequest)(nil), // 18: payload.v1.Search.MultiObjectRequest + (*Search_Config)(nil), // 19: payload.v1.Search.Config + (*Search_Response)(nil), // 20: payload.v1.Search.Response + (*Search_Responses)(nil), // 21: payload.v1.Search.Responses + (*Search_StreamResponse)(nil), // 22: payload.v1.Search.StreamResponse + (*Filter_Target)(nil), // 23: payload.v1.Filter.Target + (*Filter_Config)(nil), // 24: payload.v1.Filter.Config + (*Insert_Request)(nil), // 25: payload.v1.Insert.Request + (*Insert_MultiRequest)(nil), // 26: payload.v1.Insert.MultiRequest + (*Insert_ObjectRequest)(nil), // 27: payload.v1.Insert.ObjectRequest + (*Insert_MultiObjectRequest)(nil), // 28: payload.v1.Insert.MultiObjectRequest + (*Insert_Config)(nil), // 29: payload.v1.Insert.Config + (*Update_Request)(nil), // 30: payload.v1.Update.Request + (*Update_MultiRequest)(nil), // 31: payload.v1.Update.MultiRequest + (*Update_ObjectRequest)(nil), // 32: payload.v1.Update.ObjectRequest + (*Update_MultiObjectRequest)(nil), // 33: payload.v1.Update.MultiObjectRequest + (*Update_Config)(nil), // 34: payload.v1.Update.Config + (*Upsert_Request)(nil), // 35: payload.v1.Upsert.Request + (*Upsert_MultiRequest)(nil), // 36: payload.v1.Upsert.MultiRequest + (*Upsert_ObjectRequest)(nil), // 37: payload.v1.Upsert.ObjectRequest + (*Upsert_MultiObjectRequest)(nil), // 38: payload.v1.Upsert.MultiObjectRequest + (*Upsert_Config)(nil), // 39: payload.v1.Upsert.Config + (*Remove_Request)(nil), // 40: payload.v1.Remove.Request + (*Remove_MultiRequest)(nil), // 41: payload.v1.Remove.MultiRequest + (*Remove_TimestampRequest)(nil), // 42: payload.v1.Remove.TimestampRequest + (*Remove_Timestamp)(nil), // 43: payload.v1.Remove.Timestamp + (*Remove_Config)(nil), // 44: payload.v1.Remove.Config + (*Object_VectorRequest)(nil), // 45: payload.v1.Object.VectorRequest + (*Object_Distance)(nil), // 46: payload.v1.Object.Distance + (*Object_StreamDistance)(nil), // 47: payload.v1.Object.StreamDistance + (*Object_ID)(nil), // 48: payload.v1.Object.ID + (*Object_IDs)(nil), // 49: payload.v1.Object.IDs + (*Object_Vector)(nil), // 50: payload.v1.Object.Vector + (*Object_Vectors)(nil), // 51: payload.v1.Object.Vectors + (*Object_StreamVector)(nil), // 52: payload.v1.Object.StreamVector + (*Object_ReshapeVector)(nil), // 53: payload.v1.Object.ReshapeVector + (*Object_Blob)(nil), // 54: payload.v1.Object.Blob + (*Object_StreamBlob)(nil), // 55: payload.v1.Object.StreamBlob + (*Object_Location)(nil), // 56: payload.v1.Object.Location + (*Object_StreamLocation)(nil), // 57: payload.v1.Object.StreamLocation + (*Object_Locations)(nil), // 58: payload.v1.Object.Locations + (*Object_List)(nil), // 59: payload.v1.Object.List + (*Object_List_Request)(nil), // 60: payload.v1.Object.List.Request + (*Object_List_Response)(nil), // 61: payload.v1.Object.List.Response + (*Control_CreateIndexRequest)(nil), // 62: payload.v1.Control.CreateIndexRequest + (*Discoverer_Request)(nil), // 63: payload.v1.Discoverer.Request + (*Info_Index)(nil), // 64: payload.v1.Info.Index + (*Info_Pod)(nil), // 65: payload.v1.Info.Pod + (*Info_Node)(nil), // 66: payload.v1.Info.Node + (*Info_CPU)(nil), // 67: payload.v1.Info.CPU + (*Info_Memory)(nil), // 68: payload.v1.Info.Memory + (*Info_Pods)(nil), // 69: payload.v1.Info.Pods + (*Info_Nodes)(nil), // 70: payload.v1.Info.Nodes + (*Info_IPs)(nil), // 71: payload.v1.Info.IPs + (*Info_Index_Count)(nil), // 72: payload.v1.Info.Index.Count + (*Info_Index_UUID)(nil), // 73: payload.v1.Info.Index.UUID + (*Info_Index_UUID_Committed)(nil), // 74: payload.v1.Info.Index.UUID.Committed + (*Info_Index_UUID_Uncommitted)(nil), // 75: payload.v1.Info.Index.UUID.Uncommitted + (*status.Status)(nil), // 76: google.rpc.Status } var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{ - 20, // 0: payload.v1.Search.Request.config:type_name -> payload.v1.Search.Config - 14, // 1: payload.v1.Search.MultiRequest.requests:type_name -> payload.v1.Search.Request - 20, // 2: payload.v1.Search.IDRequest.config:type_name -> payload.v1.Search.Config - 16, // 3: payload.v1.Search.MultiIDRequest.requests:type_name -> payload.v1.Search.IDRequest - 20, // 4: payload.v1.Search.ObjectRequest.config:type_name -> payload.v1.Search.Config - 24, // 5: payload.v1.Search.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target - 18, // 6: payload.v1.Search.MultiObjectRequest.requests:type_name -> payload.v1.Search.ObjectRequest - 25, // 7: payload.v1.Search.Config.ingress_filters:type_name -> payload.v1.Filter.Config - 25, // 8: payload.v1.Search.Config.egress_filters:type_name -> payload.v1.Filter.Config + 19, // 0: payload.v1.Search.Request.config:type_name -> payload.v1.Search.Config + 13, // 1: payload.v1.Search.MultiRequest.requests:type_name -> payload.v1.Search.Request + 19, // 2: payload.v1.Search.IDRequest.config:type_name -> payload.v1.Search.Config + 15, // 3: payload.v1.Search.MultiIDRequest.requests:type_name -> payload.v1.Search.IDRequest + 19, // 4: payload.v1.Search.ObjectRequest.config:type_name -> payload.v1.Search.Config + 23, // 5: payload.v1.Search.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target + 17, // 6: payload.v1.Search.MultiObjectRequest.requests:type_name -> payload.v1.Search.ObjectRequest + 24, // 7: payload.v1.Search.Config.ingress_filters:type_name -> payload.v1.Filter.Config + 24, // 8: payload.v1.Search.Config.egress_filters:type_name -> payload.v1.Filter.Config 0, // 9: payload.v1.Search.Config.aggregation_algorithm:type_name -> payload.v1.Search.AggregationAlgorithm - 47, // 10: payload.v1.Search.Response.results:type_name -> payload.v1.Object.Distance - 21, // 11: payload.v1.Search.Responses.responses:type_name -> payload.v1.Search.Response - 21, // 12: payload.v1.Search.StreamResponse.response:type_name -> payload.v1.Search.Response - 79, // 13: payload.v1.Search.StreamResponse.status:type_name -> google.rpc.Status - 24, // 14: payload.v1.Filter.Config.targets:type_name -> payload.v1.Filter.Target - 51, // 15: payload.v1.Insert.Request.vector:type_name -> payload.v1.Object.Vector - 30, // 16: payload.v1.Insert.Request.config:type_name -> payload.v1.Insert.Config - 26, // 17: payload.v1.Insert.MultiRequest.requests:type_name -> payload.v1.Insert.Request - 55, // 18: payload.v1.Insert.ObjectRequest.object:type_name -> payload.v1.Object.Blob - 30, // 19: payload.v1.Insert.ObjectRequest.config:type_name -> payload.v1.Insert.Config - 24, // 20: payload.v1.Insert.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target - 28, // 21: payload.v1.Insert.MultiObjectRequest.requests:type_name -> payload.v1.Insert.ObjectRequest - 25, // 22: payload.v1.Insert.Config.filters:type_name -> payload.v1.Filter.Config - 51, // 23: payload.v1.Update.Request.vector:type_name -> payload.v1.Object.Vector - 35, // 24: payload.v1.Update.Request.config:type_name -> payload.v1.Update.Config - 31, // 25: payload.v1.Update.MultiRequest.requests:type_name -> payload.v1.Update.Request - 55, // 26: payload.v1.Update.ObjectRequest.object:type_name -> payload.v1.Object.Blob - 35, // 27: payload.v1.Update.ObjectRequest.config:type_name -> payload.v1.Update.Config - 24, // 28: payload.v1.Update.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target - 33, // 29: payload.v1.Update.MultiObjectRequest.requests:type_name -> payload.v1.Update.ObjectRequest - 25, // 30: payload.v1.Update.Config.filters:type_name -> payload.v1.Filter.Config - 51, // 31: payload.v1.Upsert.Request.vector:type_name -> payload.v1.Object.Vector - 40, // 32: payload.v1.Upsert.Request.config:type_name -> payload.v1.Upsert.Config - 36, // 33: payload.v1.Upsert.MultiRequest.requests:type_name -> payload.v1.Upsert.Request - 55, // 34: payload.v1.Upsert.ObjectRequest.object:type_name -> payload.v1.Object.Blob - 40, // 35: payload.v1.Upsert.ObjectRequest.config:type_name -> payload.v1.Upsert.Config - 24, // 36: payload.v1.Upsert.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target - 38, // 37: payload.v1.Upsert.MultiObjectRequest.requests:type_name -> payload.v1.Upsert.ObjectRequest - 25, // 38: payload.v1.Upsert.Config.filters:type_name -> payload.v1.Filter.Config - 49, // 39: payload.v1.Remove.Request.id:type_name -> payload.v1.Object.ID - 45, // 40: payload.v1.Remove.Request.config:type_name -> payload.v1.Remove.Config - 41, // 41: payload.v1.Remove.MultiRequest.requests:type_name -> payload.v1.Remove.Request - 44, // 42: payload.v1.Remove.TimestampRequest.timestamps:type_name -> payload.v1.Remove.Timestamp + 46, // 10: payload.v1.Search.Response.results:type_name -> payload.v1.Object.Distance + 20, // 11: payload.v1.Search.Responses.responses:type_name -> payload.v1.Search.Response + 20, // 12: payload.v1.Search.StreamResponse.response:type_name -> payload.v1.Search.Response + 76, // 13: payload.v1.Search.StreamResponse.status:type_name -> google.rpc.Status + 23, // 14: payload.v1.Filter.Config.targets:type_name -> payload.v1.Filter.Target + 50, // 15: payload.v1.Insert.Request.vector:type_name -> payload.v1.Object.Vector + 29, // 16: payload.v1.Insert.Request.config:type_name -> payload.v1.Insert.Config + 25, // 17: payload.v1.Insert.MultiRequest.requests:type_name -> payload.v1.Insert.Request + 54, // 18: payload.v1.Insert.ObjectRequest.object:type_name -> payload.v1.Object.Blob + 29, // 19: payload.v1.Insert.ObjectRequest.config:type_name -> payload.v1.Insert.Config + 23, // 20: payload.v1.Insert.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target + 27, // 21: payload.v1.Insert.MultiObjectRequest.requests:type_name -> payload.v1.Insert.ObjectRequest + 24, // 22: payload.v1.Insert.Config.filters:type_name -> payload.v1.Filter.Config + 50, // 23: payload.v1.Update.Request.vector:type_name -> payload.v1.Object.Vector + 34, // 24: payload.v1.Update.Request.config:type_name -> payload.v1.Update.Config + 30, // 25: payload.v1.Update.MultiRequest.requests:type_name -> payload.v1.Update.Request + 54, // 26: payload.v1.Update.ObjectRequest.object:type_name -> payload.v1.Object.Blob + 34, // 27: payload.v1.Update.ObjectRequest.config:type_name -> payload.v1.Update.Config + 23, // 28: payload.v1.Update.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target + 32, // 29: payload.v1.Update.MultiObjectRequest.requests:type_name -> payload.v1.Update.ObjectRequest + 24, // 30: payload.v1.Update.Config.filters:type_name -> payload.v1.Filter.Config + 50, // 31: payload.v1.Upsert.Request.vector:type_name -> payload.v1.Object.Vector + 39, // 32: payload.v1.Upsert.Request.config:type_name -> payload.v1.Upsert.Config + 35, // 33: payload.v1.Upsert.MultiRequest.requests:type_name -> payload.v1.Upsert.Request + 54, // 34: payload.v1.Upsert.ObjectRequest.object:type_name -> payload.v1.Object.Blob + 39, // 35: payload.v1.Upsert.ObjectRequest.config:type_name -> payload.v1.Upsert.Config + 23, // 36: payload.v1.Upsert.ObjectRequest.vectorizer:type_name -> payload.v1.Filter.Target + 37, // 37: payload.v1.Upsert.MultiObjectRequest.requests:type_name -> payload.v1.Upsert.ObjectRequest + 24, // 38: payload.v1.Upsert.Config.filters:type_name -> payload.v1.Filter.Config + 48, // 39: payload.v1.Remove.Request.id:type_name -> payload.v1.Object.ID + 44, // 40: payload.v1.Remove.Request.config:type_name -> payload.v1.Remove.Config + 40, // 41: payload.v1.Remove.MultiRequest.requests:type_name -> payload.v1.Remove.Request + 43, // 42: payload.v1.Remove.TimestampRequest.timestamps:type_name -> payload.v1.Remove.Timestamp 1, // 43: payload.v1.Remove.Timestamp.operator:type_name -> payload.v1.Remove.Timestamp.Operator - 49, // 44: payload.v1.Object.VectorRequest.id:type_name -> payload.v1.Object.ID - 25, // 45: payload.v1.Object.VectorRequest.filters:type_name -> payload.v1.Filter.Config - 47, // 46: payload.v1.Object.StreamDistance.distance:type_name -> payload.v1.Object.Distance - 79, // 47: payload.v1.Object.StreamDistance.status:type_name -> google.rpc.Status - 51, // 48: payload.v1.Object.Vectors.vectors:type_name -> payload.v1.Object.Vector - 51, // 49: payload.v1.Object.StreamVector.vector:type_name -> payload.v1.Object.Vector - 79, // 50: payload.v1.Object.StreamVector.status:type_name -> google.rpc.Status - 55, // 51: payload.v1.Object.StreamBlob.blob:type_name -> payload.v1.Object.Blob - 79, // 52: payload.v1.Object.StreamBlob.status:type_name -> google.rpc.Status - 57, // 53: payload.v1.Object.StreamLocation.location:type_name -> payload.v1.Object.Location - 79, // 54: payload.v1.Object.StreamLocation.status:type_name -> google.rpc.Status - 57, // 55: payload.v1.Object.Locations.locations:type_name -> payload.v1.Object.Location - 51, // 56: payload.v1.Object.List.Response.vector:type_name -> payload.v1.Object.Vector - 79, // 57: payload.v1.Object.List.Response.status:type_name -> google.rpc.Status - 68, // 58: payload.v1.Info.Pod.cpu:type_name -> payload.v1.Info.CPU - 69, // 59: payload.v1.Info.Pod.memory:type_name -> payload.v1.Info.Memory - 67, // 60: payload.v1.Info.Pod.node:type_name -> payload.v1.Info.Node - 68, // 61: payload.v1.Info.Node.cpu:type_name -> payload.v1.Info.CPU - 69, // 62: payload.v1.Info.Node.memory:type_name -> payload.v1.Info.Memory - 70, // 63: payload.v1.Info.Node.Pods:type_name -> payload.v1.Info.Pods - 66, // 64: payload.v1.Info.Pods.pods:type_name -> payload.v1.Info.Pod - 67, // 65: payload.v1.Info.Nodes.nodes:type_name -> payload.v1.Info.Node - 77, // 66: payload.v1.Mirror.Targets.targets:type_name -> payload.v1.Mirror.Target - 67, // [67:67] is the sub-list for method output_type - 67, // [67:67] is the sub-list for method input_type - 67, // [67:67] is the sub-list for extension type_name - 67, // [67:67] is the sub-list for extension extendee - 0, // [0:67] is the sub-list for field type_name + 48, // 44: payload.v1.Object.VectorRequest.id:type_name -> payload.v1.Object.ID + 24, // 45: payload.v1.Object.VectorRequest.filters:type_name -> payload.v1.Filter.Config + 46, // 46: payload.v1.Object.StreamDistance.distance:type_name -> payload.v1.Object.Distance + 76, // 47: payload.v1.Object.StreamDistance.status:type_name -> google.rpc.Status + 50, // 48: payload.v1.Object.Vectors.vectors:type_name -> payload.v1.Object.Vector + 50, // 49: payload.v1.Object.StreamVector.vector:type_name -> payload.v1.Object.Vector + 76, // 50: payload.v1.Object.StreamVector.status:type_name -> google.rpc.Status + 54, // 51: payload.v1.Object.StreamBlob.blob:type_name -> payload.v1.Object.Blob + 76, // 52: payload.v1.Object.StreamBlob.status:type_name -> google.rpc.Status + 56, // 53: payload.v1.Object.StreamLocation.location:type_name -> payload.v1.Object.Location + 76, // 54: payload.v1.Object.StreamLocation.status:type_name -> google.rpc.Status + 56, // 55: payload.v1.Object.Locations.locations:type_name -> payload.v1.Object.Location + 50, // 56: payload.v1.Object.List.Response.vector:type_name -> payload.v1.Object.Vector + 76, // 57: payload.v1.Object.List.Response.status:type_name -> google.rpc.Status + 67, // 58: payload.v1.Info.Pod.cpu:type_name -> payload.v1.Info.CPU + 68, // 59: payload.v1.Info.Pod.memory:type_name -> payload.v1.Info.Memory + 66, // 60: payload.v1.Info.Pod.node:type_name -> payload.v1.Info.Node + 67, // 61: payload.v1.Info.Node.cpu:type_name -> payload.v1.Info.CPU + 68, // 62: payload.v1.Info.Node.memory:type_name -> payload.v1.Info.Memory + 69, // 63: payload.v1.Info.Node.Pods:type_name -> payload.v1.Info.Pods + 65, // 64: payload.v1.Info.Pods.pods:type_name -> payload.v1.Info.Pod + 66, // 65: payload.v1.Info.Nodes.nodes:type_name -> payload.v1.Info.Node + 66, // [66:66] is the sub-list for method output_type + 66, // [66:66] is the sub-list for method input_type + 66, // [66:66] is the sub-list for extension type_name + 66, // [66:66] is the sub-list for extension extendee + 0, // [0:66] is the sub-list for field type_name } func init() { file_apis_proto_v1_payload_payload_proto_init() } @@ -5279,18 +5121,6 @@ func file_apis_proto_v1_payload_payload_proto_init() { } } file_apis_proto_v1_payload_payload_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Mirror); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_apis_proto_v1_payload_payload_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Empty); i { case 0: return &v.state @@ -5302,7 +5132,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_Request); i { case 0: return &v.state @@ -5314,7 +5144,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_MultiRequest); i { case 0: return &v.state @@ -5326,7 +5156,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_IDRequest); i { case 0: return &v.state @@ -5338,7 +5168,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_MultiIDRequest); i { case 0: return &v.state @@ -5350,7 +5180,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_ObjectRequest); i { case 0: return &v.state @@ -5362,7 +5192,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_MultiObjectRequest); i { case 0: return &v.state @@ -5374,7 +5204,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_Config); i { case 0: return &v.state @@ -5386,7 +5216,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_Response); i { case 0: return &v.state @@ -5398,7 +5228,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_Responses); i { case 0: return &v.state @@ -5410,7 +5240,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Search_StreamResponse); i { case 0: return &v.state @@ -5422,7 +5252,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Filter_Target); i { case 0: return &v.state @@ -5434,7 +5264,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Filter_Config); i { case 0: return &v.state @@ -5446,7 +5276,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Insert_Request); i { case 0: return &v.state @@ -5458,7 +5288,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Insert_MultiRequest); i { case 0: return &v.state @@ -5470,7 +5300,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Insert_ObjectRequest); i { case 0: return &v.state @@ -5482,7 +5312,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Insert_MultiObjectRequest); i { case 0: return &v.state @@ -5494,7 +5324,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Insert_Config); i { case 0: return &v.state @@ -5506,7 +5336,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Update_Request); i { case 0: return &v.state @@ -5518,7 +5348,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Update_MultiRequest); i { case 0: return &v.state @@ -5530,7 +5360,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Update_ObjectRequest); i { case 0: return &v.state @@ -5542,7 +5372,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Update_MultiObjectRequest); i { case 0: return &v.state @@ -5554,7 +5384,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Update_Config); i { case 0: return &v.state @@ -5566,7 +5396,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Upsert_Request); i { case 0: return &v.state @@ -5578,7 +5408,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Upsert_MultiRequest); i { case 0: return &v.state @@ -5590,7 +5420,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Upsert_ObjectRequest); i { case 0: return &v.state @@ -5602,7 +5432,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Upsert_MultiObjectRequest); i { case 0: return &v.state @@ -5614,7 +5444,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Upsert_Config); i { case 0: return &v.state @@ -5626,7 +5456,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Remove_Request); i { case 0: return &v.state @@ -5638,7 +5468,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Remove_MultiRequest); i { case 0: return &v.state @@ -5650,7 +5480,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Remove_TimestampRequest); i { case 0: return &v.state @@ -5662,7 +5492,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Remove_Timestamp); i { case 0: return &v.state @@ -5674,7 +5504,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Remove_Config); i { case 0: return &v.state @@ -5686,7 +5516,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_VectorRequest); i { case 0: return &v.state @@ -5698,7 +5528,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_Distance); i { case 0: return &v.state @@ -5710,7 +5540,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_StreamDistance); i { case 0: return &v.state @@ -5722,7 +5552,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_ID); i { case 0: return &v.state @@ -5734,7 +5564,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_IDs); i { case 0: return &v.state @@ -5746,7 +5576,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_Vector); i { case 0: return &v.state @@ -5758,7 +5588,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_Vectors); i { case 0: return &v.state @@ -5770,7 +5600,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_StreamVector); i { case 0: return &v.state @@ -5782,7 +5612,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_ReshapeVector); i { case 0: return &v.state @@ -5794,7 +5624,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_Blob); i { case 0: return &v.state @@ -5806,7 +5636,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_StreamBlob); i { case 0: return &v.state @@ -5818,7 +5648,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_Location); i { case 0: return &v.state @@ -5830,7 +5660,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_StreamLocation); i { case 0: return &v.state @@ -5842,7 +5672,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_Locations); i { case 0: return &v.state @@ -5854,7 +5684,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_List); i { case 0: return &v.state @@ -5866,7 +5696,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_List_Request); i { case 0: return &v.state @@ -5878,7 +5708,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Object_List_Response); i { case 0: return &v.state @@ -5890,7 +5720,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Control_CreateIndexRequest); i { case 0: return &v.state @@ -5902,7 +5732,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Discoverer_Request); i { case 0: return &v.state @@ -5914,7 +5744,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Index); i { case 0: return &v.state @@ -5926,7 +5756,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Pod); i { case 0: return &v.state @@ -5938,7 +5768,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Node); i { case 0: return &v.state @@ -5950,7 +5780,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_CPU); i { case 0: return &v.state @@ -5962,7 +5792,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Memory); i { case 0: return &v.state @@ -5974,7 +5804,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Pods); i { case 0: return &v.state @@ -5986,7 +5816,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Nodes); i { case 0: return &v.state @@ -5998,7 +5828,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_IPs); i { case 0: return &v.state @@ -6010,7 +5840,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Index_Count); i { case 0: return &v.state @@ -6022,7 +5852,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Index_UUID); i { case 0: return &v.state @@ -6034,7 +5864,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Index_UUID_Committed); i { case 0: return &v.state @@ -6046,7 +5876,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Index_UUID_Uncommitted); i { case 0: return &v.state @@ -6058,52 +5888,28 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Mirror_Target); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_apis_proto_v1_payload_payload_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Mirror_Targets); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } } - file_apis_proto_v1_payload_payload_proto_msgTypes[21].OneofWrappers = []interface{}{ + file_apis_proto_v1_payload_payload_proto_msgTypes[20].OneofWrappers = []interface{}{ (*Search_StreamResponse_Response)(nil), (*Search_StreamResponse_Status)(nil), } - file_apis_proto_v1_payload_payload_proto_msgTypes[46].OneofWrappers = []interface{}{ + file_apis_proto_v1_payload_payload_proto_msgTypes[45].OneofWrappers = []interface{}{ (*Object_StreamDistance_Distance)(nil), (*Object_StreamDistance_Status)(nil), } - file_apis_proto_v1_payload_payload_proto_msgTypes[51].OneofWrappers = []interface{}{ + file_apis_proto_v1_payload_payload_proto_msgTypes[50].OneofWrappers = []interface{}{ (*Object_StreamVector_Vector)(nil), (*Object_StreamVector_Status)(nil), } - file_apis_proto_v1_payload_payload_proto_msgTypes[54].OneofWrappers = []interface{}{ + file_apis_proto_v1_payload_payload_proto_msgTypes[53].OneofWrappers = []interface{}{ (*Object_StreamBlob_Blob)(nil), (*Object_StreamBlob_Status)(nil), } - file_apis_proto_v1_payload_payload_proto_msgTypes[56].OneofWrappers = []interface{}{ + file_apis_proto_v1_payload_payload_proto_msgTypes[55].OneofWrappers = []interface{}{ (*Object_StreamLocation_Location)(nil), (*Object_StreamLocation_Status)(nil), } - file_apis_proto_v1_payload_payload_proto_msgTypes[60].OneofWrappers = []interface{}{ + file_apis_proto_v1_payload_payload_proto_msgTypes[59].OneofWrappers = []interface{}{ (*Object_List_Response_Vector)(nil), (*Object_List_Response_Status)(nil), } @@ -6113,7 +5919,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_apis_proto_v1_payload_payload_proto_rawDesc, NumEnums: 2, - NumMessages: 77, + NumMessages: 74, NumExtensions: 0, NumServices: 0, }, diff --git a/apis/grpc/v1/rpc/errdetails/error_details.pb.go b/apis/grpc/v1/rpc/errdetails/error_details.pb.go index 35dbdbc7cb..cb886e1056 100644 --- a/apis/grpc/v1/rpc/errdetails/error_details.pb.go +++ b/apis/grpc/v1/rpc/errdetails/error_details.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/rpc/error_details.proto package errdetails diff --git a/apis/grpc/v1/vald/filter.pb.go b/apis/grpc/v1/vald/filter.pb.go index 6941e5957e..445d914de8 100644 --- a/apis/grpc/v1/vald/filter.pb.go +++ b/apis/grpc/v1/vald/filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/vald/filter.proto package vald diff --git a/apis/grpc/v1/vald/insert.pb.go b/apis/grpc/v1/vald/insert.pb.go index 97c175a8ed..12e9cdadcb 100644 --- a/apis/grpc/v1/vald/insert.pb.go +++ b/apis/grpc/v1/vald/insert.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/vald/insert.proto package vald diff --git a/apis/grpc/v1/vald/object.pb.go b/apis/grpc/v1/vald/object.pb.go index 86fba70815..82038cb45b 100644 --- a/apis/grpc/v1/vald/object.pb.go +++ b/apis/grpc/v1/vald/object.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/vald/object.proto package vald diff --git a/apis/grpc/v1/vald/remove.pb.go b/apis/grpc/v1/vald/remove.pb.go index 77a04d56f2..cdd4498fd6 100644 --- a/apis/grpc/v1/vald/remove.pb.go +++ b/apis/grpc/v1/vald/remove.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/vald/remove.proto package vald diff --git a/apis/grpc/v1/vald/search.pb.go b/apis/grpc/v1/vald/search.pb.go index 317502f6ad..b0882e457c 100644 --- a/apis/grpc/v1/vald/search.pb.go +++ b/apis/grpc/v1/vald/search.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/vald/search.proto package vald diff --git a/apis/grpc/v1/vald/update.pb.go b/apis/grpc/v1/vald/update.pb.go index 4de4679e6e..2ba6c96aad 100644 --- a/apis/grpc/v1/vald/update.pb.go +++ b/apis/grpc/v1/vald/update.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/vald/update.proto package vald diff --git a/apis/grpc/v1/vald/upsert.pb.go b/apis/grpc/v1/vald/upsert.pb.go index 0305d2a0a2..9996c63db2 100644 --- a/apis/grpc/v1/vald/upsert.pb.go +++ b/apis/grpc/v1/vald/upsert.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.23.4 +// protoc v4.24.4 // source: apis/proto/v1/vald/upsert.proto package vald diff --git a/charts/vald-helm-operator/crds/valdrelease.yaml b/charts/vald-helm-operator/crds/valdrelease.yaml index ec63d9b334..da3f7666c2 100644 --- a/charts/vald-helm-operator/crds/valdrelease.yaml +++ b/charts/vald-helm-operator/crds/valdrelease.yaml @@ -2032,11 +2032,15 @@ spec: type: object properties: egress: - type: object - x-kubernetes-preserve-unknown-fields: true + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true ingress: - type: object - x-kubernetes-preserve-unknown-fields: true + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true enabled: type: boolean observability: @@ -7124,6 +7128,971 @@ spec: annotations: type: object x-kubernetes-preserve-unknown-fields: true + corrector: + type: object + properties: + agent_namespace: + type: string + bbolt_async_write_concurrency: + type: integer + minimum: 1 + discoverer: + type: object + properties: + agent_client_options: + type: object + properties: + addrs: + type: array + items: + type: string + backoff: + type: object + properties: + backoff_factor: + type: number + backoff_time_limit: + type: string + enable_error_log: + type: boolean + initial_duration: + type: string + jitter_limit: + type: string + maximum_duration: + type: string + retry_count: + type: integer + call_option: + type: object + x-kubernetes-preserve-unknown-fields: true + circuit_breaker: + type: object + properties: + closed_error_rate: + type: number + closed_refresh_timeout: + type: string + half_open_error_rate: + type: number + min_samples: + type: integer + open_timeout: + type: string + connection_pool: + type: object + properties: + enable_dns_resolver: + type: boolean + enable_rebalance: + type: boolean + old_conn_close_duration: + type: string + rebalance_duration: + type: string + size: + type: integer + dial_option: + type: object + properties: + backoff_base_delay: + type: string + backoff_jitter: + type: number + backoff_max_delay: + type: string + backoff_multiplier: + type: number + enable_backoff: + type: boolean + initial_connection_window_size: + type: integer + initial_window_size: + type: integer + insecure: + type: boolean + interceptors: + type: array + items: + type: string + enum: + - TraceInterceptor + keepalive: + type: object + properties: + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_msg_size: + type: integer + min_connection_timeout: + type: string + net: + type: object + properties: + dialer: + type: object + properties: + dual_stack_enabled: + type: boolean + keepalive: + type: string + timeout: + type: string + dns: + type: object + properties: + cache_enabled: + type: boolean + cache_expiration: + type: string + refresh_duration: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + read_buffer_size: + type: integer + timeout: + type: string + write_buffer_size: + type: integer + health_check_duration: + type: string + max_recv_msg_size: + type: integer + max_retry_rpc_buffer_size: + type: integer + max_send_msg_size: + type: integer + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + wait_for_ready: + type: boolean + client: + type: object + properties: + addrs: + type: array + items: + type: string + backoff: + type: object + properties: + backoff_factor: + type: number + backoff_time_limit: + type: string + enable_error_log: + type: boolean + initial_duration: + type: string + jitter_limit: + type: string + maximum_duration: + type: string + retry_count: + type: integer + call_option: + type: object + x-kubernetes-preserve-unknown-fields: true + circuit_breaker: + type: object + properties: + closed_error_rate: + type: number + closed_refresh_timeout: + type: string + half_open_error_rate: + type: number + min_samples: + type: integer + open_timeout: + type: string + connection_pool: + type: object + properties: + enable_dns_resolver: + type: boolean + enable_rebalance: + type: boolean + old_conn_close_duration: + type: string + rebalance_duration: + type: string + size: + type: integer + dial_option: + type: object + properties: + backoff_base_delay: + type: string + backoff_jitter: + type: number + backoff_max_delay: + type: string + backoff_multiplier: + type: number + enable_backoff: + type: boolean + initial_connection_window_size: + type: integer + initial_window_size: + type: integer + insecure: + type: boolean + interceptors: + type: array + items: + type: string + enum: + - TraceInterceptor + keepalive: + type: object + properties: + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_msg_size: + type: integer + min_connection_timeout: + type: string + net: + type: object + properties: + dialer: + type: object + properties: + dual_stack_enabled: + type: boolean + keepalive: + type: string + timeout: + type: string + dns: + type: object + properties: + cache_enabled: + type: boolean + cache_expiration: + type: string + refresh_duration: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + read_buffer_size: + type: integer + timeout: + type: string + write_buffer_size: + type: integer + health_check_duration: + type: string + max_recv_msg_size: + type: integer + max_retry_rpc_buffer_size: + type: integer + max_send_msg_size: + type: integer + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + wait_for_ready: + type: boolean + duration: + type: string + enabled: + type: boolean + env: + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true + image: + type: object + properties: + pullPolicy: + type: string + enum: + - Always + - Never + - IfNotPresent + repository: + type: string + tag: + type: string + name: + type: string + node_name: + type: string + observability: + type: object + properties: + enabled: + type: boolean + metrics: + type: object + properties: + enable_cgo: + type: boolean + enable_goroutine: + type: boolean + enable_memory: + type: boolean + enable_version_info: + type: boolean + version_info_labels: + type: array + items: + type: string + enum: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - cgo_enabled + - ngt_version + - build_cpu_info_flags + otlp: + type: object + properties: + attribute: + type: object + properties: + namespace: + type: string + node_name: + type: string + pod_name: + type: string + service_name: + type: string + collector_endpoint: + type: string + metrics_export_interval: + type: string + metrics_export_timeout: + type: string + trace_batch_timeout: + type: string + trace_export_timeout: + type: string + trace_max_export_batch_size: + type: integer + trace_max_queue_size: + type: integer + trace: + type: object + properties: + enabled: + type: boolean + schedule: + type: string + server_config: + type: object + properties: + full_shutdown_duration: + type: string + healths: + type: object + properties: + liveness: + type: object + properties: + enabled: + type: boolean + host: + type: string + livenessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + readiness: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + readinessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + startup: + type: object + properties: + enabled: + type: boolean + port: + type: integer + maximum: 65535 + minimum: 0 + startupProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + metrics: + type: object + properties: + pprof: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + servers: + type: object + properties: + grpc: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + grpc: + type: object + properties: + bidirectional_stream_concurrency: + type: integer + connection_timeout: + type: string + enable_reflection: + type: boolean + header_table_size: + type: integer + initial_conn_window_size: + type: integer + initial_window_size: + type: integer + interceptors: + type: array + items: + type: string + enum: + - RecoverInterceptor + - AccessLogInterceptor + - TraceInterceptor + - MetricInterceptor + keepalive: + type: object + properties: + max_conn_age: + type: string + max_conn_age_grace: + type: string + max_conn_idle: + type: string + min_time: + type: string + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_header_list_size: + type: integer + max_receive_message_size: + type: integer + max_send_message_size: + type: integer + read_buffer_size: + type: integer + write_buffer_size: + type: integer + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + restart: + type: boolean + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + rest: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + startingDeadlineSeconds: + type: integer + stream_list_concurrency: + type: integer + minimum: 1 + suspend: + type: boolean + ttlSecondsAfterFinished: + type: integer + version: + type: string + pattern: ^v[0-9]+\.[0-9]+\.[0-9]$ enabled: type: boolean env: diff --git a/charts/vald-helm-operator/templates/clusterrole.yaml b/charts/vald-helm-operator/templates/clusterrole.yaml index 50af049435..560e78ed49 100644 --- a/charts/vald-helm-operator/templates/clusterrole.yaml +++ b/charts/vald-helm-operator/templates/clusterrole.yaml @@ -187,4 +187,17 @@ rules: - get - patch - update + - apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + {{- end }} diff --git a/charts/vald/README.md b/charts/vald/README.md index 7f77ca110a..9cadef374f 100644 --- a/charts/vald/README.md +++ b/charts/vald/README.md @@ -703,6 +703,26 @@ Run the following command to install the chart, | manager.index.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity preferred scheduling terms | | manager.index.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution | list | `[]` | pod anti-affinity required scheduling terms | | manager.index.annotations | object | `{}` | deployment annotations | +| manager.index.corrector.agent_namespace | string | `"_MY_POD_NAMESPACE_"` | namespace of agent pods to manage | +| manager.index.corrector.bbolt_async_write_concurrency | int | `2048` | concurrency for bbolt async write | +| manager.index.corrector.discoverer.agent_client_options | object | `{"dial_option":{"net":{"dialer":{"keepalive":"15m"}}}}` | gRPC client options for agents (overrides defaults.grpc.client) | +| manager.index.corrector.discoverer.client | object | `{}` | gRPC client for discoverer (overrides defaults.grpc.client) | +| manager.index.corrector.discoverer.duration | string | `"500ms"` | refresh duration to discover | +| manager.index.corrector.enabled | bool | `false` | enable index correction CronJob | +| manager.index.corrector.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | +| manager.index.corrector.image.pullPolicy | string | `"Always"` | | +| manager.index.corrector.image.repository | string | `"vdaas/vald-index-correction"` | image repository | +| manager.index.corrector.image.tag | string | `""` | image tag (overrides defaults.image.tag) | +| manager.index.corrector.name | string | `"vald-index-correction"` | name of index correction job | +| manager.index.corrector.node_name | string | `""` | node name | +| manager.index.corrector.observability | object | `{"otlp":{"attribute":{"service_name":"vald-manager-index"}}}` | observability config (overrides defaults.observability) | +| manager.index.corrector.schedule | string | `"6 3 * * *"` | CronJob schedule setting for index correction | +| manager.index.corrector.server_config | object | `{"healths":{"liveness":{},"readiness":{},"startup":{}},"metrics":{"pprof":{}},"servers":{"grpc":{},"rest":{}}}` | server config (overrides defaults.server_config) | +| manager.index.corrector.startingDeadlineSeconds | int | `86400` | startingDeadlineSeconds setting for K8s completed jobs | +| manager.index.corrector.stream_list_concurrency | int | `200` | concurrency for stream list object rpc | +| manager.index.corrector.suspend | bool | `false` | CronJob suspend setting for index correction | +| manager.index.corrector.ttlSecondsAfterFinished | int | `86400` | ttl setting for K8s completed jobs | +| manager.index.corrector.version | string | `"v0.0.0"` | version of index manager config | | manager.index.enabled | bool | `true` | index manager enabled | | manager.index.env | list | `[{"name":"MY_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"MY_POD_NAME","valueFrom":{"fieldRef":{"fieldPath":"metadata.name"}}},{"name":"MY_POD_NAMESPACE","valueFrom":{"fieldRef":{"fieldPath":"metadata.namespace"}}}]` | environment variables | | manager.index.externalTrafficPolicy | string | `""` | external traffic policy (can be specified when service type is LoadBalancer or NodePort) : Cluster or Local | diff --git a/charts/vald/templates/agent/networkpolicy.yaml b/charts/vald/templates/agent/networkpolicy.yaml index e6e1ef90bb..ef51ea5439 100644 --- a/charts/vald/templates/agent/networkpolicy.yaml +++ b/charts/vald/templates/agent/networkpolicy.yaml @@ -17,6 +17,7 @@ {{- $agent := .Values.agent -}} {{- $lb := .Values.gateway.lb -}} {{- $index := .Values.manager.index -}} +{{- $corrector := .Values.manager.index.corrector -}} {{- if .Values.defaults.networkPolicy.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -44,6 +45,12 @@ spec: podSelector: matchLabels: app: {{ $index.name }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + matchLabels: + app: {{ $corrector.name }} {{- if .Values.defaults.networkPolicy.custom.ingress }} {{- toYaml .Values.defaults.networkPolicy.custom.ingress | nindent 4 }} {{- end }} diff --git a/charts/vald/templates/discoverer/networkpolicy.yaml b/charts/vald/templates/discoverer/networkpolicy.yaml index 065cc97b59..41803702dd 100644 --- a/charts/vald/templates/discoverer/networkpolicy.yaml +++ b/charts/vald/templates/discoverer/networkpolicy.yaml @@ -17,6 +17,7 @@ {{- $discoverer := .Values.discoverer -}} {{- $lb := .Values.gateway.lb -}} {{- $index := .Values.manager.index -}} +{{- $corrector := .Values.manager.index.corrector -}} {{- if .Values.defaults.networkPolicy.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -46,6 +47,12 @@ spec: podSelector: matchLabels: app: {{ $index.name }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + matchLabels: + app: {{ $corrector.name }} {{- if .Values.defaults.networkPolicy.custom.ingress }} {{- toYaml .Values.defaults.networkPolicy.custom.ingress | nindent 4 }} {{- end }} diff --git a/charts/vald/templates/index/job/correction/configmap.yaml b/charts/vald/templates/index/job/correction/configmap.yaml new file mode 100644 index 0000000000..77db8328e2 --- /dev/null +++ b/charts/vald/templates/index/job/correction/configmap.yaml @@ -0,0 +1,71 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- $corrector := .Values.manager.index.corrector -}} +{{- $gateway := .Values.gateway.lb -}} +{{- $index := .Values.manager.index -}} +{{- $agent := .Values.agent -}} +{{- $discoverer := .Values.discoverer -}} +{{- if $corrector.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $corrector.name }}-config + labels: + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} + app.kubernetes.io/component: {{ $corrector.name }} +data: + config.yaml: | + --- + version: {{ $corrector.version }} + time_zone: {{ default .Values.defaults.time_zone $corrector.time_zone }} + logging: + {{- $logging := dict "Values" $corrector.logging "default" .Values.defaults.logging }} + {{- include "vald.logging" $logging | nindent 6 }} + server_config: + {{- $servers := dict "Values" $corrector.server_config "default" .Values.defaults.server_config }} + {{- include "vald.servers" $servers | nindent 6 }} + observability: + {{- $observability := dict "Values" $corrector.observability "default" .Values.defaults.observability }} + {{- include "vald.observability" $observability | nindent 6 }} + corrector: + agent_port: {{ default .Values.defaults.server_config.servers.grpc.port $agent.server_config.servers.grpc.port }} + agent_name: {{ $agent.name | quote }} + agent_dns: {{ $agent.name }}.{{ .Release.Namespace }}.svc.cluster.local + agent_namespace: {{ $index.indexer.agent_namespace | quote }} + node_name: {{ $index.indexer.node_name | quote }} + stream_list_concurrency: {{ $corrector.stream_list_concurrency }} + bbolt_async_write_concurrency: {{ $corrector.bbolt_async_write_concurrency }} + index_replica: {{ $gateway.gateway_config.index_replica }} + discoverer: + duration: {{ $corrector.discoverer.duration }} + client: + {{- $discovererClient := $index.indexer.discoverer.client }} + {{- $discovererServerPort := $discoverer.server_config.servers.grpc.port }} + {{- $defaultDiscovererHost := printf "%s.%s.svc.cluster.local" $discoverer.name .Release.Namespace }} + {{- $defaultDiscovererPort := default .Values.defaults.server_config.servers.grpc.port $discovererServerPort }} + {{- $defaultDiscovererAddr := (list (printf "%s:%d" $defaultDiscovererHost (int64 $defaultDiscovererPort))) }} + {{- $discovererAddrs := dict "Values" $discovererClient.addrs "default" $defaultDiscovererAddr }} + {{- include "vald.grpc.client.addrs" $discovererAddrs | nindent 10 }} + {{- $discovererGRPCclient := dict "Values" $discovererClient "default" .Values.defaults.grpc.client }} + {{- include "vald.grpc.client" $discovererGRPCclient | nindent 10 }} + agent_client_options: + {{- include "vald.grpc.client.addrs" (dict "Values" $corrector.discoverer.agent_client_options.addrs) | nindent 10 }} + {{- include "vald.grpc.client" (dict "Values" $corrector.discoverer.agent_client_options "default" .Values.defaults.grpc.client) | nindent 10 }} +{{- end }} diff --git a/charts/vald/templates/index/job/correction/cronjob.yaml b/charts/vald/templates/index/job/correction/cronjob.yaml new file mode 100644 index 0000000000..70c3f408f5 --- /dev/null +++ b/charts/vald/templates/index/job/correction/cronjob.yaml @@ -0,0 +1,62 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- $corrector := .Values.manager.index.corrector -}} +{{- if $corrector.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ $corrector.name }} + labels: + app: {{ $corrector.name }} + app.kubernetes.io/name: {{ include "vald.name" . }} + helm.sh/chart: {{ include "vald.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.Version }} +spec: + schedule: {{ $corrector.schedule }} + concurrencyPolicy: Forbid + suspend: {{ $corrector.suspend }} + startingDeadlineSeconds: {{ $corrector.startingDeadlineSeconds }} + jobTemplate: + spec: + ttlSecondsAfterFinished: {{ $corrector.ttlSecondsAfterFinished }} + template: + metadata: + labels: + app: {{ $corrector.name }} + spec: + containers: + - name: {{ $corrector.name }} + image: "{{ $corrector.image.repository }}:{{ default .Values.defaults.image.tag $corrector.image.tag }}" + imagePullPolicy: {{ $corrector.image.pullPolicy }} + volumeMounts: + - name: {{ $corrector.name }}-config + mountPath: /etc/server/ + {{- $servers := dict "Values" $corrector.server_config "default" .Values.defaults.server_config -}} + {{- include "vald.containerPorts" $servers | trim | nindent 14 }} + {{- if $corrector.env }} + env: + {{- toYaml $corrector.env | nindent 16 }} + {{- end }} + restartPolicy: OnFailure + volumes: + - name: {{ $corrector.name }}-config + configMap: + defaultMode: 420 + name: {{ $corrector.name }}-config +{{- end }} diff --git a/charts/vald/values.schema.json b/charts/vald/values.schema.json index e987b81837..076f7440a6 100644 --- a/charts/vald/values.schema.json +++ b/charts/vald/values.schema.json @@ -3288,12 +3288,14 @@ "description": "custom network policies that a user can add", "properties": { "egress": { - "type": "object", - "description": "custom egress network policies that a user can add" + "type": "array", + "description": "custom egress network policies that a user can add", + "items": { "type": "object" } }, "ingress": { - "type": "object", - "description": "custom ingress network policies that a user can add" + "type": "array", + "description": "custom ingress network policies that a user can add", + "items": { "type": "object" } } } }, @@ -11838,6 +11840,1723 @@ "type": "object", "description": "deployment annotations" }, + "corrector": { + "type": "object", + "properties": { + "agent_namespace": { + "type": "string", + "description": "namespace of agent pods to manage" + }, + "bbolt_async_write_concurrency": { + "type": "integer", + "description": "concurrency for bbolt async write", + "minimum": 1 + }, + "discoverer": { + "type": "object", + "properties": { + "agent_client_options": { + "type": "object", + "properties": { + "addrs": { + "type": "array", + "description": "gRPC client addresses", + "items": { "type": "string" } + }, + "backoff": { + "type": "object", + "properties": { + "backoff_factor": { + "type": "number", + "description": "gRPC client backoff factor" + }, + "backoff_time_limit": { + "type": "string", + "description": "gRPC client backoff time limit" + }, + "enable_error_log": { + "type": "boolean", + "description": "gRPC client backoff log enabled" + }, + "initial_duration": { + "type": "string", + "description": "gRPC client backoff initial duration" + }, + "jitter_limit": { + "type": "string", + "description": "gRPC client backoff jitter limit" + }, + "maximum_duration": { + "type": "string", + "description": "gRPC client backoff maximum duration" + }, + "retry_count": { + "type": "integer", + "description": "gRPC client backoff retry count" + } + } + }, + "call_option": { "type": "object" }, + "circuit_breaker": { + "type": "object", + "properties": { + "closed_error_rate": { + "type": "number", + "description": "gRPC client circuitbreaker closed error rate" + }, + "closed_refresh_timeout": { + "type": "string", + "description": "gRPC client circuitbreaker closed refresh timeout" + }, + "half_open_error_rate": { + "type": "number", + "description": "gRPC client circuitbreaker half-open error rate" + }, + "min_samples": { + "type": "integer", + "description": "gRPC client circuitbreaker minimum sampling count" + }, + "open_timeout": { + "type": "string", + "description": "gRPC client circuitbreaker open timeout" + } + } + }, + "connection_pool": { + "type": "object", + "properties": { + "enable_dns_resolver": { + "type": "boolean", + "description": "enables gRPC client connection pool dns resolver, when enabled vald uses ip handshake exclude dns discovery which improves network performance" + }, + "enable_rebalance": { + "type": "boolean", + "description": "enables gRPC client connection pool rebalance" + }, + "old_conn_close_duration": { + "type": "string", + "description": "makes delay before gRPC client connection closing during connection pool rebalance" + }, + "rebalance_duration": { + "type": "string", + "description": "gRPC client connection pool rebalance duration" + }, + "size": { + "type": "integer", + "description": "gRPC client connection pool size" + } + } + }, + "dial_option": { + "type": "object", + "properties": { + "backoff_base_delay": { + "type": "string", + "description": "gRPC client dial option base backoff delay" + }, + "backoff_jitter": { + "type": "number", + "description": "gRPC client dial option base backoff delay" + }, + "backoff_max_delay": { + "type": "string", + "description": "gRPC client dial option max backoff delay" + }, + "backoff_multiplier": { + "type": "number", + "description": "gRPC client dial option base backoff delay" + }, + "enable_backoff": { + "type": "boolean", + "description": "gRPC client dial option backoff enabled" + }, + "initial_connection_window_size": { + "type": "integer", + "description": "gRPC client dial option initial connection window size" + }, + "initial_window_size": { + "type": "integer", + "description": "gRPC client dial option initial window size" + }, + "insecure": { + "type": "boolean", + "description": "gRPC client dial option insecure enabled" + }, + "interceptors": { + "type": "array", + "description": "gRPC client interceptors", + "items": { + "type": "string", + "enum": ["TraceInterceptor"] + } + }, + "keepalive": { + "type": "object", + "properties": { + "permit_without_stream": { + "type": "boolean", + "description": "gRPC client keep alive permit without stream" + }, + "time": { + "type": "string", + "description": "gRPC client keep alive time" + }, + "timeout": { + "type": "string", + "description": "gRPC client keep alive timeout" + } + } + }, + "max_msg_size": { + "type": "integer", + "description": "gRPC client dial option max message size" + }, + "min_connection_timeout": { + "type": "string", + "description": "gRPC client dial option minimum connection timeout" + }, + "net": { + "type": "object", + "properties": { + "dialer": { + "type": "object", + "properties": { + "dual_stack_enabled": { + "type": "boolean", + "description": "gRPC client TCP dialer dual stack enabled" + }, + "keepalive": { + "type": "string", + "description": "gRPC client TCP dialer keep alive" + }, + "timeout": { + "type": "string", + "description": "gRPC client TCP dialer timeout" + } + } + }, + "dns": { + "type": "object", + "properties": { + "cache_enabled": { + "type": "boolean", + "description": "gRPC client TCP DNS cache enabled" + }, + "cache_expiration": { + "type": "string", + "description": "gRPC client TCP DNS cache expiration" + }, + "refresh_duration": { + "type": "string", + "description": "gRPC client TCP DNS cache refresh duration" + } + } + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "tls": { + "type": "object", + "properties": { + "ca": { + "type": "string", + "description": "TLS ca path" + }, + "cert": { + "type": "string", + "description": "TLS cert path" + }, + "enabled": { + "type": "boolean", + "description": "TLS enabled" + }, + "insecure_skip_verify": { + "type": "boolean", + "description": "enable/disable skip SSL certificate verification" + }, + "key": { + "type": "string", + "description": "TLS key path" + } + } + } + } + }, + "read_buffer_size": { + "type": "integer", + "description": "gRPC client dial option read buffer size" + }, + "timeout": { + "type": "string", + "description": "gRPC client dial option timeout" + }, + "write_buffer_size": { + "type": "integer", + "description": "gRPC client dial option write buffer size" + } + } + }, + "health_check_duration": { + "type": "string", + "description": "gRPC client health check duration" + }, + "max_recv_msg_size": { "type": "integer" }, + "max_retry_rpc_buffer_size": { "type": "integer" }, + "max_send_msg_size": { "type": "integer" }, + "tls": { + "type": "object", + "properties": { + "ca": { + "type": "string", + "description": "TLS ca path" + }, + "cert": { + "type": "string", + "description": "TLS cert path" + }, + "enabled": { + "type": "boolean", + "description": "TLS enabled" + }, + "insecure_skip_verify": { + "type": "boolean", + "description": "enable/disable skip SSL certificate verification" + }, + "key": { + "type": "string", + "description": "TLS key path" + } + } + }, + "wait_for_ready": { "type": "boolean" } + } + }, + "client": { + "type": "object", + "properties": { + "addrs": { + "type": "array", + "description": "gRPC client addresses", + "items": { "type": "string" } + }, + "backoff": { + "type": "object", + "properties": { + "backoff_factor": { + "type": "number", + "description": "gRPC client backoff factor" + }, + "backoff_time_limit": { + "type": "string", + "description": "gRPC client backoff time limit" + }, + "enable_error_log": { + "type": "boolean", + "description": "gRPC client backoff log enabled" + }, + "initial_duration": { + "type": "string", + "description": "gRPC client backoff initial duration" + }, + "jitter_limit": { + "type": "string", + "description": "gRPC client backoff jitter limit" + }, + "maximum_duration": { + "type": "string", + "description": "gRPC client backoff maximum duration" + }, + "retry_count": { + "type": "integer", + "description": "gRPC client backoff retry count" + } + } + }, + "call_option": { "type": "object" }, + "circuit_breaker": { + "type": "object", + "properties": { + "closed_error_rate": { + "type": "number", + "description": "gRPC client circuitbreaker closed error rate" + }, + "closed_refresh_timeout": { + "type": "string", + "description": "gRPC client circuitbreaker closed refresh timeout" + }, + "half_open_error_rate": { + "type": "number", + "description": "gRPC client circuitbreaker half-open error rate" + }, + "min_samples": { + "type": "integer", + "description": "gRPC client circuitbreaker minimum sampling count" + }, + "open_timeout": { + "type": "string", + "description": "gRPC client circuitbreaker open timeout" + } + } + }, + "connection_pool": { + "type": "object", + "properties": { + "enable_dns_resolver": { + "type": "boolean", + "description": "enables gRPC client connection pool dns resolver, when enabled vald uses ip handshake exclude dns discovery which improves network performance" + }, + "enable_rebalance": { + "type": "boolean", + "description": "enables gRPC client connection pool rebalance" + }, + "old_conn_close_duration": { + "type": "string", + "description": "makes delay before gRPC client connection closing during connection pool rebalance" + }, + "rebalance_duration": { + "type": "string", + "description": "gRPC client connection pool rebalance duration" + }, + "size": { + "type": "integer", + "description": "gRPC client connection pool size" + } + } + }, + "dial_option": { + "type": "object", + "properties": { + "backoff_base_delay": { + "type": "string", + "description": "gRPC client dial option base backoff delay" + }, + "backoff_jitter": { + "type": "number", + "description": "gRPC client dial option base backoff delay" + }, + "backoff_max_delay": { + "type": "string", + "description": "gRPC client dial option max backoff delay" + }, + "backoff_multiplier": { + "type": "number", + "description": "gRPC client dial option base backoff delay" + }, + "enable_backoff": { + "type": "boolean", + "description": "gRPC client dial option backoff enabled" + }, + "initial_connection_window_size": { + "type": "integer", + "description": "gRPC client dial option initial connection window size" + }, + "initial_window_size": { + "type": "integer", + "description": "gRPC client dial option initial window size" + }, + "insecure": { + "type": "boolean", + "description": "gRPC client dial option insecure enabled" + }, + "interceptors": { + "type": "array", + "description": "gRPC client interceptors", + "items": { + "type": "string", + "enum": ["TraceInterceptor"] + } + }, + "keepalive": { + "type": "object", + "properties": { + "permit_without_stream": { + "type": "boolean", + "description": "gRPC client keep alive permit without stream" + }, + "time": { + "type": "string", + "description": "gRPC client keep alive time" + }, + "timeout": { + "type": "string", + "description": "gRPC client keep alive timeout" + } + } + }, + "max_msg_size": { + "type": "integer", + "description": "gRPC client dial option max message size" + }, + "min_connection_timeout": { + "type": "string", + "description": "gRPC client dial option minimum connection timeout" + }, + "net": { + "type": "object", + "properties": { + "dialer": { + "type": "object", + "properties": { + "dual_stack_enabled": { + "type": "boolean", + "description": "gRPC client TCP dialer dual stack enabled" + }, + "keepalive": { + "type": "string", + "description": "gRPC client TCP dialer keep alive" + }, + "timeout": { + "type": "string", + "description": "gRPC client TCP dialer timeout" + } + } + }, + "dns": { + "type": "object", + "properties": { + "cache_enabled": { + "type": "boolean", + "description": "gRPC client TCP DNS cache enabled" + }, + "cache_expiration": { + "type": "string", + "description": "gRPC client TCP DNS cache expiration" + }, + "refresh_duration": { + "type": "string", + "description": "gRPC client TCP DNS cache refresh duration" + } + } + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "tls": { + "type": "object", + "properties": { + "ca": { + "type": "string", + "description": "TLS ca path" + }, + "cert": { + "type": "string", + "description": "TLS cert path" + }, + "enabled": { + "type": "boolean", + "description": "TLS enabled" + }, + "insecure_skip_verify": { + "type": "boolean", + "description": "enable/disable skip SSL certificate verification" + }, + "key": { + "type": "string", + "description": "TLS key path" + } + } + } + } + }, + "read_buffer_size": { + "type": "integer", + "description": "gRPC client dial option read buffer size" + }, + "timeout": { + "type": "string", + "description": "gRPC client dial option timeout" + }, + "write_buffer_size": { + "type": "integer", + "description": "gRPC client dial option write buffer size" + } + } + }, + "health_check_duration": { + "type": "string", + "description": "gRPC client health check duration" + }, + "max_recv_msg_size": { "type": "integer" }, + "max_retry_rpc_buffer_size": { "type": "integer" }, + "max_send_msg_size": { "type": "integer" }, + "tls": { + "type": "object", + "properties": { + "ca": { + "type": "string", + "description": "TLS ca path" + }, + "cert": { + "type": "string", + "description": "TLS cert path" + }, + "enabled": { + "type": "boolean", + "description": "TLS enabled" + }, + "insecure_skip_verify": { + "type": "boolean", + "description": "enable/disable skip SSL certificate verification" + }, + "key": { + "type": "string", + "description": "TLS key path" + } + } + }, + "wait_for_ready": { "type": "boolean" } + } + }, + "duration": { + "type": "string", + "description": "refresh duration to discover" + } + } + }, + "enabled": { + "type": "boolean", + "description": "enable index correction CronJob" + }, + "env": { + "type": "array", + "description": "environment variables", + "items": { "type": "object" } + }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "type": "string", + "description": "image pull policy", + "enum": ["Always", "Never", "IfNotPresent"] + }, + "repository": { + "type": "string", + "description": "image repository" + }, + "tag": { + "type": "string", + "description": "image tag (overrides defaults.image.tag)" + } + } + }, + "name": { + "type": "string", + "description": "name of index correction job" + }, + "node_name": { "type": "string", "description": "node name" }, + "observability": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "observability features enabled" + }, + "metrics": { + "type": "object", + "properties": { + "enable_cgo": { + "type": "boolean", + "description": "CGO metrics enabled" + }, + "enable_goroutine": { + "type": "boolean", + "description": "goroutine metrics enabled" + }, + "enable_memory": { + "type": "boolean", + "description": "memory metrics enabled" + }, + "enable_version_info": { + "type": "boolean", + "description": "version info metrics enabled" + }, + "version_info_labels": { + "type": "array", + "description": "enabled label names of version info", + "items": { + "type": "string", + "enum": [ + "vald_version", + "server_name", + "git_commit", + "build_time", + "go_version", + "go_os", + "go_arch", + "cgo_enabled", + "ngt_version", + "build_cpu_info_flags" + ] + } + } + } + }, + "otlp": { + "type": "object", + "properties": { + "attribute": { + "type": "object", + "description": "default resource attribute", + "properties": { + "namespace": { + "type": "string", + "description": "namespace" + }, + "node_name": { + "type": "string", + "description": "node name" + }, + "pod_name": { + "type": "string", + "description": "pod name" + }, + "service_name": { + "type": "string", + "description": "service name" + } + } + }, + "collector_endpoint": { + "type": "string", + "description": "OpenTelemetry Collector endpoint" + }, + "metrics_export_interval": { + "type": "string", + "description": "metrics export interval" + }, + "metrics_export_timeout": { + "type": "string", + "description": "metrics export timeout" + }, + "trace_batch_timeout": { + "type": "string", + "description": "trace batch timeout" + }, + "trace_export_timeout": { + "type": "string", + "description": "trace export timeout" + }, + "trace_max_export_batch_size": { + "type": "integer", + "description": "trace maximum export batch size" + }, + "trace_max_queue_size": { + "type": "integer", + "description": "trace maximum queue size" + } + } + }, + "trace": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "trace enabled" + } + } + } + } + }, + "schedule": { + "type": "string", + "description": "CronJob schedule setting for index correction" + }, + "server_config": { + "type": "object", + "properties": { + "full_shutdown_duration": { + "type": "string", + "description": "server full shutdown duration" + }, + "healths": { + "type": "object", + "properties": { + "liveness": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "liveness server enabled" + }, + "host": { + "type": "string", + "description": "liveness server host" + }, + "livenessProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer", + "description": "liveness probe failure threshold" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "liveness probe path" + }, + "port": { + "type": "string", + "description": "liveness probe port" + }, + "scheme": { + "type": "string", + "description": "liveness probe scheme" + } + } + }, + "initialDelaySeconds": { + "type": "integer", + "description": "liveness probe initial delay seconds" + }, + "periodSeconds": { + "type": "integer", + "description": "liveness probe period seconds" + }, + "successThreshold": { + "type": "integer", + "description": "liveness probe success threshold" + }, + "timeoutSeconds": { + "type": "integer", + "description": "liveness probe timeout seconds" + } + } + }, + "port": { + "type": "integer", + "description": "liveness server port", + "maximum": 65535, + "minimum": 0 + }, + "server": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "handler_timeout": { + "type": "string", + "description": "REST server handler timeout" + }, + "idle_timeout": { + "type": "string", + "description": "REST server idle timeout" + }, + "read_header_timeout": { + "type": "string", + "description": "REST server read header timeout" + }, + "read_timeout": { + "type": "string", + "description": "REST server read timeout" + }, + "shutdown_duration": { + "type": "string", + "description": "REST server shutdown duration" + }, + "write_timeout": { + "type": "string", + "description": "REST server write timeout" + } + } + }, + "mode": { + "type": "string", + "description": "REST server server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "REST server probe wait time" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "liveness server service port", + "maximum": 65535, + "minimum": 0 + } + } + }, + "readiness": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "readiness server enabled" + }, + "host": { + "type": "string", + "description": "readiness server host" + }, + "port": { + "type": "integer", + "description": "readiness server port", + "maximum": 65535, + "minimum": 0 + }, + "readinessProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer", + "description": "readiness probe failure threshold" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "readiness probe path" + }, + "port": { + "type": "string", + "description": "readiness probe port" + }, + "scheme": { + "type": "string", + "description": "readiness probe scheme" + } + } + }, + "initialDelaySeconds": { + "type": "integer", + "description": "readiness probe initial delay seconds" + }, + "periodSeconds": { + "type": "integer", + "description": "readiness probe period seconds" + }, + "successThreshold": { + "type": "integer", + "description": "readiness probe success threshold" + }, + "timeoutSeconds": { + "type": "integer", + "description": "readiness probe timeout seconds" + } + } + }, + "server": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "handler_timeout": { + "type": "string", + "description": "REST server handler timeout" + }, + "idle_timeout": { + "type": "string", + "description": "REST server idle timeout" + }, + "read_header_timeout": { + "type": "string", + "description": "REST server read header timeout" + }, + "read_timeout": { + "type": "string", + "description": "REST server read timeout" + }, + "shutdown_duration": { + "type": "string", + "description": "REST server shutdown duration" + }, + "write_timeout": { + "type": "string", + "description": "REST server write timeout" + } + } + }, + "mode": { + "type": "string", + "description": "REST server server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "REST server probe wait time" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "readiness server service port", + "maximum": 65535, + "minimum": 0 + } + } + }, + "startup": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "startup server enabled" + }, + "port": { + "type": "integer", + "description": "startup server port", + "maximum": 65535, + "minimum": 0 + }, + "startupProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer", + "description": "startup probe failure threshold" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "startup probe path" + }, + "port": { + "type": "string", + "description": "startup probe port" + }, + "scheme": { + "type": "string", + "description": "startup probe scheme" + } + } + }, + "initialDelaySeconds": { + "type": "integer", + "description": "startup probe initial delay seconds" + }, + "periodSeconds": { + "type": "integer", + "description": "startup probe period seconds" + }, + "successThreshold": { + "type": "integer", + "description": "startup probe success threshold" + }, + "timeoutSeconds": { + "type": "integer", + "description": "startup probe timeout seconds" + } + } + } + } + } + } + }, + "metrics": { + "type": "object", + "properties": { + "pprof": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "pprof server enabled" + }, + "host": { + "type": "string", + "description": "pprof server host" + }, + "port": { + "type": "integer", + "description": "pprof server port", + "maximum": 65535, + "minimum": 0 + }, + "server": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "handler_timeout": { + "type": "string", + "description": "REST server handler timeout" + }, + "idle_timeout": { + "type": "string", + "description": "REST server idle timeout" + }, + "read_header_timeout": { + "type": "string", + "description": "REST server read header timeout" + }, + "read_timeout": { + "type": "string", + "description": "REST server read timeout" + }, + "shutdown_duration": { + "type": "string", + "description": "REST server shutdown duration" + }, + "write_timeout": { + "type": "string", + "description": "REST server write timeout" + } + } + }, + "mode": { + "type": "string", + "description": "REST server server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "REST server probe wait time" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "pprof server service port", + "maximum": 65535, + "minimum": 0 + } + } + } + } + }, + "servers": { + "type": "object", + "properties": { + "grpc": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "gRPC server enabled" + }, + "host": { + "type": "string", + "description": "gRPC server host" + }, + "port": { + "type": "integer", + "description": "gRPC server port", + "maximum": 65535, + "minimum": 0 + }, + "server": { + "type": "object", + "properties": { + "grpc": { + "type": "object", + "properties": { + "bidirectional_stream_concurrency": { + "type": "integer", + "description": "gRPC server bidirectional stream concurrency" + }, + "connection_timeout": { + "type": "string", + "description": "gRPC server connection timeout" + }, + "enable_reflection": { + "type": "boolean", + "description": "gRPC server reflection option" + }, + "header_table_size": { + "type": "integer", + "description": "gRPC server header table size" + }, + "initial_conn_window_size": { + "type": "integer", + "description": "gRPC server initial connection window size" + }, + "initial_window_size": { + "type": "integer", + "description": "gRPC server initial window size" + }, + "interceptors": { + "type": "array", + "description": "gRPC server interceptors", + "items": { + "type": "string", + "enum": [ + "RecoverInterceptor", + "AccessLogInterceptor", + "TraceInterceptor", + "MetricInterceptor" + ] + } + }, + "keepalive": { + "type": "object", + "properties": { + "max_conn_age": { + "type": "string", + "description": "gRPC server keep alive max connection age" + }, + "max_conn_age_grace": { + "type": "string", + "description": "gRPC server keep alive max connection age grace" + }, + "max_conn_idle": { + "type": "string", + "description": "gRPC server keep alive max connection idle" + }, + "min_time": { + "type": "string", + "description": "gRPC server keep alive min_time" + }, + "permit_without_stream": { + "type": "boolean", + "description": "gRPC server keep alive permit_without_stream" + }, + "time": { + "type": "string", + "description": "gRPC server keep alive time" + }, + "timeout": { + "type": "string", + "description": "gRPC server keep alive timeout" + } + } + }, + "max_header_list_size": { + "type": "integer", + "description": "gRPC server max header list size" + }, + "max_receive_message_size": { + "type": "integer", + "description": "gRPC server max receive message size" + }, + "max_send_message_size": { + "type": "integer", + "description": "gRPC server max send message size" + }, + "read_buffer_size": { + "type": "integer", + "description": "gRPC server read buffer size" + }, + "write_buffer_size": { + "type": "integer", + "description": "gRPC server write buffer size" + } + } + }, + "mode": { + "type": "string", + "description": "gRPC server server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "gRPC server probe wait time" + }, + "restart": { + "type": "boolean", + "description": "gRPC server restart" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "gRPC server service port", + "maximum": 65535, + "minimum": 0 + } + } + }, + "rest": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "REST server enabled" + }, + "host": { + "type": "string", + "description": "REST server host" + }, + "port": { + "type": "integer", + "description": "REST server port", + "maximum": 65535, + "minimum": 0 + }, + "server": { + "type": "object", + "properties": { + "http": { + "type": "object", + "properties": { + "handler_timeout": { + "type": "string", + "description": "REST server handler timeout" + }, + "idle_timeout": { + "type": "string", + "description": "REST server idle timeout" + }, + "read_header_timeout": { + "type": "string", + "description": "REST server read header timeout" + }, + "read_timeout": { + "type": "string", + "description": "REST server read timeout" + }, + "shutdown_duration": { + "type": "string", + "description": "REST server shutdown duration" + }, + "write_timeout": { + "type": "string", + "description": "REST server write timeout" + } + } + }, + "mode": { + "type": "string", + "description": "REST server server mode" + }, + "network": { + "type": "string", + "description": "mysql network", + "enum": [ + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + "unix", + "unixgram", + "unixpacket" + ] + }, + "probe_wait_time": { + "type": "string", + "description": "REST server probe wait time" + }, + "socket_option": { + "type": "object", + "properties": { + "ip_recover_destination_addr": { + "type": "boolean", + "description": "server listen socket option for ip_recover_destination_addr functionality" + }, + "ip_transparent": { + "type": "boolean", + "description": "server listen socket option for ip_transparent functionality" + }, + "reuse_addr": { + "type": "boolean", + "description": "server listen socket option for reuse_addr functionality" + }, + "reuse_port": { + "type": "boolean", + "description": "server listen socket option for reuse_port functionality" + }, + "tcp_cork": { + "type": "boolean", + "description": "server listen socket option for tcp_cork functionality" + }, + "tcp_defer_accept": { + "type": "boolean", + "description": "server listen socket option for tcp_defer_accept functionality" + }, + "tcp_fast_open": { + "type": "boolean", + "description": "server listen socket option for tcp_fast_open functionality" + }, + "tcp_no_delay": { + "type": "boolean", + "description": "server listen socket option for tcp_no_delay functionality" + }, + "tcp_quick_ack": { + "type": "boolean", + "description": "server listen socket option for tcp_quick_ack functionality" + } + } + }, + "socket_path": { + "type": "string", + "description": "mysql socket_path" + } + } + }, + "servicePort": { + "type": "integer", + "description": "REST server service port", + "maximum": 65535, + "minimum": 0 + } + } + } + } + }, + "tls": { + "type": "object", + "properties": { + "ca": { + "type": "string", + "description": "TLS ca path" + }, + "cert": { + "type": "string", + "description": "TLS cert path" + }, + "enabled": { + "type": "boolean", + "description": "TLS enabled" + }, + "insecure_skip_verify": { + "type": "boolean", + "description": "enable/disable skip SSL certificate verification" + }, + "key": { + "type": "string", + "description": "TLS key path" + } + } + } + } + }, + "startingDeadlineSeconds": { + "type": "integer", + "description": "startingDeadlineSeconds setting for K8s completed jobs" + }, + "stream_list_concurrency": { + "type": "integer", + "description": "concurrency for stream list object rpc", + "minimum": 1 + }, + "suspend": { + "type": "boolean", + "description": "CronJob suspend setting for index correction" + }, + "ttlSecondsAfterFinished": { + "type": "integer", + "description": "ttl setting for K8s completed jobs" + }, + "version": { + "type": "string", + "description": "version of gateway config", + "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]$" + } + } + }, "enabled": { "type": "boolean", "description": "index manager enabled" diff --git a/charts/vald/values.yaml b/charts/vald/values.yaml index 5149940109..68f84f0a37 100644 --- a/charts/vald/values.yaml +++ b/charts/vald/values.yaml @@ -838,12 +838,12 @@ defaults: # @schema {"name": "defaults.networkPolicy.custom", "type": "object"} # defaults.networkPolicy.custom -- custom network policies that a user can add custom: - # @schema {"name": "defaults.networkPolicy.custom.ingress", "type": "object"} + # @schema {"name": "defaults.networkPolicy.custom.ingress", "type": "array", "items": {"type": "object"}} # defaults.networkPolicy.custom.ingress -- custom ingress network policies that a user can add - ingress: {} - # @schema {"name": "defaults.networkPolicy.custom.egress", "type": "object"} + ingress: [] + # @schema {"name": "defaults.networkPolicy.custom.egress", "type": "array", "items": {"type": "object"}} # defaults.networkPolicy.custom.egress -- custom egress network policies that a user can add - egress: {} + egress: [] # @schema {"name": "gateway", "type": "object"} gateway: # @schema {"name": "gateway.lb", "type": "object"} @@ -2974,3 +2974,94 @@ manager: net: dialer: keepalive: 15m #indexer fetches uncommitted index length, which includes huge payload so we need to set keepalive longer than usual + # @schema {"name": "manager.index.corrector", "type": "object"} + corrector: + # @schema {"name": "manager.index.corrector.name", "type": "string"} + # manager.index.corrector.name -- name of index correction job + name: vald-index-correction + # @schema {"name": "manager.index.corrector.image", "alias": "image"} + image: + # manager.index.corrector.image.repository -- image repository + repository: vdaas/vald-index-correction + # manager.index.corrector.image.tag -- image tag (overrides defaults.image.tag) + tag: "" + # manager.index.image.pullPolicy -- image pull policy + pullPolicy: Always + # @schema {"name": "manager.index.corrector.server_config", "alias": "server_config"} + # manager.index.corrector.server_config -- server config (overrides defaults.server_config) + server_config: + servers: + rest: {} + grpc: {} + healths: + liveness: {} + readiness: {} + startup: {} + metrics: + pprof: {} + # @schema {"name": "manager.index.corrector.env", "alias": "env"} + # manager.index.corrector.env -- environment variables + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # @schema {"name": "manager.index.corrector.observability", "alias": "observability"} + # manager.index.corrector.observability -- observability config (overrides defaults.observability) + observability: + otlp: + attribute: + service_name: vald-index-correction + # @schema {"name": "manager.index.corrector.enabled", "type": "boolean"} + # manager.index.corrector.enabled -- enable index correction CronJob + enabled: false + # @schema {"name": "manager.index.corrector.schedule", "type": "string"} + # manager.index.corrector.schedule -- CronJob schedule setting for index correction + schedule: "6 3 * * *" + # @schema {"name": "manager.index.corrector.suspend", "type": "boolean"} + # manager.index.corrector.suspend -- CronJob suspend setting for index correction + suspend: false + # @schema {"name": "manager.index.corrector.ttlSecondsAfterFinished", "type": "integer"} + # manager.index.corrector.ttlSecondsAfterFinished -- ttl setting for K8s completed jobs + ttlSecondsAfterFinished: 86400 + # @schema {"name": "manager.index.corrector.startingDeadlineSeconds", "type": "integer"} + # manager.index.corrector.startingDeadlineSeconds -- startingDeadlineSeconds setting for K8s completed jobs + startingDeadlineSeconds: 86400 + # @schema {"name": "manager.index.corrector.version", "alias": "version"} + # manager.index.corrector.version -- version of index manager config + version: v0.0.0 + # @schema {"name": "manager.index.corrector.stream_list_concurrency", "type": "integer", "minimum": 1} + # manager.index.corrector.stream_list_concurrency -- concurrency for stream list object rpc + stream_list_concurrency: 200 + # @schema {"name": "manager.index.corrector.bbolt_async_write_concurrency", "type": "integer", "minimum": 1} + # manager.index.corrector.bbolt_async_write_concurrency -- concurrency for bbolt async write + bbolt_async_write_concurrency: 2048 + # @schema {"name": "manager.index.corrector.agent_namespace", "type": "string"} + # manager.index.corrector.agent_namespace -- namespace of agent pods to manage + agent_namespace: _MY_POD_NAMESPACE_ + # @schema {"name": "manager.index.corrector.node_name", "type": "string"} + # manager.index.corrector.node_name -- node name + node_name: "" # _MY_NODE_NAME_ + # @schema {"name": "manager.index.corrector.discoverer", "type": "object"} + discoverer: + # @schema {"name": "manager.index.corrector.discoverer.duration", "type": "string"} + # manager.index.corrector.discoverer.duration -- refresh duration to discover + duration: 500ms + # @schema {"name": "manager.index.corrector.discoverer.client", "alias": "grpc.client"} + # manager.index.corrector.discoverer.client -- gRPC client for discoverer (overrides defaults.grpc.client) + client: {} + # @schema {"name": "manager.index.corrector.discoverer.agent_client_options", "alias": "grpc.client"} + # manager.index.corrector.discoverer.agent_client_options -- gRPC client options for agents (overrides defaults.grpc.client) + agent_client_options: + dial_option: + net: + dialer: + keepalive: 15m #indexer fetches uncommitted index length, which includes huge payload so we need to set keepalive longer than usual diff --git a/cmd/index/job/correction/main.go b/cmd/index/job/correction/main.go new file mode 100644 index 0000000000..d549626691 --- /dev/null +++ b/cmd/index/job/correction/main.go @@ -0,0 +1,59 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "context" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/pkg/index/job/correction/config" + "github.com/vdaas/vald/pkg/index/job/correction/usecase" +) + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "index correction job" +) + +func main() { + if err := safety.RecoverFunc(func() error { + return runner.Do( + context.Background(), + runner.WithName(name), + runner.WithVersion(info.Version, maxVersion, minVersion), + runner.WithConfigLoader(func(path string) (interface{}, *config.GlobalConfig, error) { + cfg, err := config.NewConfig(path) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") + } + return cfg, &cfg.GlobalConfig, nil + }), + runner.WithDaemonInitializer(func(cfg interface{}) (runner.Runner, error) { + c, ok := cfg.(*config.Data) + if !ok { + return nil, errors.ErrInvalidConfig + } + return usecase.New(c) + }), + ) + })(); err != nil { + log.Fatal(err, info.Get()) + return + } +} diff --git a/cmd/index/job/correction/sample.yaml b/cmd/index/job/correction/sample.yaml new file mode 100644 index 0000000000..bc3f144e41 --- /dev/null +++ b/cmd/index/job/correction/sample.yaml @@ -0,0 +1,233 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +version: v0.0.0 +time_zone: JST +logging: + format: raw + level: info + logger: glg +server_config: + servers: + - name: grpc + host: 0.0.0.0 + port: 8081 + grpc: + bidirectional_stream_concurrency: 20 + connection_timeout: "" + header_table_size: 0 + initial_conn_window_size: 0 + initial_window_size: 0 + interceptors: [] + keepalive: + max_conn_age: "" + max_conn_age_grace: "" + max_conn_idle: "" + time: "" + timeout: "" + max_header_list_size: 0 + max_receive_message_size: 0 + max_send_message_size: 0 + read_buffer_size: 0 + write_buffer_size: 0 + mode: GRPC + probe_wait_time: 3s + restart: true + health_check_servers: + - name: readiness + host: 0.0.0.0 + port: 3001 + http: + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + shutdown_duration: 0s + write_timeout: "" + mode: "" + probe_wait_time: 3s + metrics_servers: + startup_strategy: + - grpc + - readiness + full_shutdown_duration: 600s + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +corrector: + agent_port: 8081 + agent_name: "vald-agent-ngt" + agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_namespace: "default" + node_name: "" + stream_list_concurrency: 200 + bbolt_async_write_concurrency: 2048 + index_replica: 3 + discoverer: + duration: 500ms + client: + addrs: + - vald-discoverer.default.svc.cluster.local:8081 + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 3s + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + call_option: + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + backoff_base_delay: 1s + backoff_jitter: 0.2 + backoff_max_delay: 120s + backoff_multiplier: 1.6 + enable_backoff: false + initial_connection_window_size: 0 + initial_window_size: 0 + insecure: true + keepalive: + permit_without_stream: false + time: "" + timeout: "" + max_msg_size: 0 + min_connection_timeout: 20s + read_buffer_size: 0 + tcp: + dialer: + dual_stack_enabled: true + keepalive: "" + timeout: "" + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + timeout: "" + write_buffer_size: 0 + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + agent_client_options: + addrs: [] + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 3s + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + call_option: + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + write_buffer_size: 0 + read_buffer_size: 0 + initial_window_size: 0 + initial_connection_window_size: 0 + max_msg_size: 0 + backoff_max_delay: "120s" + backoff_base_delay: "1s" + backoff_multiplier: 1.6 + backoff_jitter: 0.2 + min_connection_timeout: "20s" + enable_backoff: false + insecure: true + timeout: "" + tcp: + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + dialer: + timeout: "" + keepalive: "15m" + dual_stack_enabled: true + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + keepalive: + permit_without_stream: false + time: "" + timeout: "" + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +observability: + enabled: false + otlp: + collector_endpoint: "otel-collector.monitoring.svc.cluster.local:4317" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-index-correction" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - ngt_version + trace: + enabled: true diff --git a/cmd/index/job/creation/main.go b/cmd/index/job/creation/main.go new file mode 100644 index 0000000000..8b63241135 --- /dev/null +++ b/cmd/index/job/creation/main.go @@ -0,0 +1,59 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "context" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/pkg/index/job/creation/config" + "github.com/vdaas/vald/pkg/index/job/creation/usecase" +) + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "index creation job" +) + +func main() { + if err := safety.RecoverFunc(func() error { + return runner.Do( + context.Background(), + runner.WithName(name), + runner.WithVersion(info.Version, maxVersion, minVersion), + runner.WithConfigLoader(func(path string) (interface{}, *config.GlobalConfig, error) { + cfg, err := config.NewConfig(path) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") + } + return cfg, &cfg.GlobalConfig, nil + }), + runner.WithDaemonInitializer(func(cfg interface{}) (runner.Runner, error) { + c, ok := cfg.(*config.Data) + if !ok { + return nil, errors.ErrInvalidConfig + } + return usecase.New(c) + }), + ) + })(); err != nil { + log.Fatal(err, info.Get()) + return + } +} diff --git a/cmd/index/job/creation/sample.yaml b/cmd/index/job/creation/sample.yaml new file mode 100644 index 0000000000..74302f6d68 --- /dev/null +++ b/cmd/index/job/creation/sample.yaml @@ -0,0 +1,231 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +version: v0.0.0 +time_zone: JST +logging: + format: raw + level: info + logger: glg +server_config: + servers: + - name: grpc + host: 0.0.0.0 + port: 8081 + grpc: + bidirectional_stream_concurrency: 20 + connection_timeout: "" + header_table_size: 0 + initial_conn_window_size: 0 + initial_window_size: 0 + interceptors: [] + keepalive: + max_conn_age: "" + max_conn_age_grace: "" + max_conn_idle: "" + time: "" + timeout: "" + max_header_list_size: 0 + max_receive_message_size: 0 + max_send_message_size: 0 + read_buffer_size: 0 + write_buffer_size: 0 + mode: GRPC + probe_wait_time: 3s + restart: true + health_check_servers: + - name: readiness + host: 0.0.0.0 + port: 3001 + http: + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + shutdown_duration: 0s + write_timeout: "" + mode: "" + probe_wait_time: 3s + metrics_servers: + startup_strategy: + - grpc + - readiness + full_shutdown_duration: 600s + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +creator: + agent_port: 8081 + agent_name: "vald-agent-ngt" + agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_namespace: "default" + node_name: "" + concurrency: 1 + discoverer: + duration: 500ms + client: + addrs: + - vald-discoverer.default.svc.cluster.local:8081 + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 3s + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + call_option: + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + backoff_base_delay: 1s + backoff_jitter: 0.2 + backoff_max_delay: 120s + backoff_multiplier: 1.6 + enable_backoff: false + initial_connection_window_size: 0 + initial_window_size: 0 + insecure: true + keepalive: + permit_without_stream: false + time: "" + timeout: "" + max_msg_size: 0 + min_connection_timeout: 20s + read_buffer_size: 0 + tcp: + dialer: + dual_stack_enabled: true + keepalive: "" + timeout: "" + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + timeout: "" + write_buffer_size: 0 + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + agent_client_options: + addrs: [] + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 3s + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + call_option: + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + write_buffer_size: 0 + read_buffer_size: 0 + initial_window_size: 0 + initial_connection_window_size: 0 + max_msg_size: 0 + backoff_max_delay: "120s" + backoff_base_delay: "1s" + backoff_multiplier: 1.6 + backoff_jitter: 0.2 + min_connection_timeout: "20s" + enable_backoff: false + insecure: true + timeout: "" + tcp: + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + dialer: + timeout: "" + keepalive: "15m" + dual_stack_enabled: true + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + keepalive: + permit_without_stream: false + time: "" + timeout: "" + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +observability: + enabled: false + otlp: + collector_endpoint: "otel-collector.monitoring.svc.cluster.local:4317" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-index-creation" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - ngt_version + trace: + enabled: true diff --git a/cmd/index/job/save/main.go b/cmd/index/job/save/main.go new file mode 100644 index 0000000000..2ad5f73221 --- /dev/null +++ b/cmd/index/job/save/main.go @@ -0,0 +1,59 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "context" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/pkg/index/job/save/config" + "github.com/vdaas/vald/pkg/index/job/save/usecase" +) + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "index save job" +) + +func main() { + if err := safety.RecoverFunc(func() error { + return runner.Do( + context.Background(), + runner.WithName(name), + runner.WithVersion(info.Version, maxVersion, minVersion), + runner.WithConfigLoader(func(path string) (interface{}, *config.GlobalConfig, error) { + cfg, err := config.NewConfig(path) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") + } + return cfg, &cfg.GlobalConfig, nil + }), + runner.WithDaemonInitializer(func(cfg interface{}) (runner.Runner, error) { + c, ok := cfg.(*config.Data) + if !ok { + return nil, errors.ErrInvalidConfig + } + return usecase.New(c) + }), + ) + })(); err != nil { + log.Fatal(err, info.Get()) + return + } +} diff --git a/cmd/index/job/save/sample.yaml b/cmd/index/job/save/sample.yaml new file mode 100644 index 0000000000..7c2cd85a79 --- /dev/null +++ b/cmd/index/job/save/sample.yaml @@ -0,0 +1,231 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +version: v0.0.0 +time_zone: JST +logging: + format: raw + level: info + logger: glg +server_config: + servers: + - name: grpc + host: 0.0.0.0 + port: 8081 + grpc: + bidirectional_stream_concurrency: 20 + connection_timeout: "" + header_table_size: 0 + initial_conn_window_size: 0 + initial_window_size: 0 + interceptors: [] + keepalive: + max_conn_age: "" + max_conn_age_grace: "" + max_conn_idle: "" + time: "" + timeout: "" + max_header_list_size: 0 + max_receive_message_size: 0 + max_send_message_size: 0 + read_buffer_size: 0 + write_buffer_size: 0 + mode: GRPC + probe_wait_time: 3s + restart: true + health_check_servers: + - name: readiness + host: 0.0.0.0 + port: 3001 + http: + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + shutdown_duration: 0s + write_timeout: "" + mode: "" + probe_wait_time: 3s + metrics_servers: + startup_strategy: + - grpc + - readiness + full_shutdown_duration: 600s + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +saver: + agent_port: 8081 + agent_name: "vald-agent-ngt" + agent_dns: vald-agent-ngt.default.svc.cluster.local + agent_namespace: "default" + node_name: "" + concurrency: 1 + discoverer: + duration: 500ms + client: + addrs: + - vald-discoverer.default.svc.cluster.local:8081 + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 3s + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + call_option: + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + backoff_base_delay: 1s + backoff_jitter: 0.2 + backoff_max_delay: 120s + backoff_multiplier: 1.6 + enable_backoff: false + initial_connection_window_size: 0 + initial_window_size: 0 + insecure: true + keepalive: + permit_without_stream: false + time: "" + timeout: "" + max_msg_size: 0 + min_connection_timeout: 20s + read_buffer_size: 0 + tcp: + dialer: + dual_stack_enabled: true + keepalive: "" + timeout: "" + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + timeout: "" + write_buffer_size: 0 + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + agent_client_options: + addrs: [] + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 3s + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + call_option: + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + write_buffer_size: 0 + read_buffer_size: 0 + initial_window_size: 0 + initial_connection_window_size: 0 + max_msg_size: 0 + backoff_max_delay: "120s" + backoff_base_delay: "1s" + backoff_multiplier: 1.6 + backoff_jitter: 0.2 + min_connection_timeout: "20s" + enable_backoff: false + insecure: true + timeout: "" + tcp: + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + dialer: + timeout: "" + keepalive: "15m" + dual_stack_enabled: true + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + keepalive: + permit_without_stream: false + time: "" + timeout: "" + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +observability: + enabled: false + otlp: + collector_endpoint: "otel-collector.monitoring.svc.cluster.local:4317" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-index-save" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - ngt_version + trace: + enabled: true diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index c967b34c55..1253743ad7 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -64,4 +64,5 @@ RUN make deps \ # additional deps RUN make k3d/install \ && make buf/install \ - && make k9s/install + && make k9s/install \ + && make minikube/install diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile new file mode 100644 index 0000000000..1938e2531f --- /dev/null +++ b/dockers/index/job/correction/Dockerfile @@ -0,0 +1,93 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG GO_VERSION=latest +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG MAINTAINER="vdaas.org vald team " + +FROM golang:${GO_VERSION} AS golang + +FROM ubuntu:devel AS builder + +ENV GO111MODULE on +ENV DEBIAN_FRONTEND noninteractive +ENV INITRD No +ENV LANG en_US.UTF-8 +ENV GOROOT /opt/go +ENV GOPATH /go +ENV PATH ${PATH}:${GOROOT}/bin:${GOPATH}/bin +ENV ORG vdaas +ENV REPO vald +ENV PKG index/job/correction +ENV APP_NAME index-correction + +# skipcq: DOK-DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + curl \ + upx \ + git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=golang /usr/local/go $GOROOT +RUN mkdir -p "$GOPATH/src" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY Makefile . +COPY .git . +COPY go.mod . +COPY go.sum . + +RUN make go/download + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal +COPY internal . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc +COPY apis/grpc . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} +COPY pkg/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +COPY cmd/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +RUN cp sample.yaml /tmp/config.yaml + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer="${MAINTAINER}" + +ENV APP_NAME index-correction + +COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +COPY --from=builder /tmp/config.yaml /etc/server/config.yaml + +USER nonroot:nonroot + +ENTRYPOINT ["/go/bin/index-correction"] diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile new file mode 100644 index 0000000000..f17fb867f9 --- /dev/null +++ b/dockers/index/job/creation/Dockerfile @@ -0,0 +1,93 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG GO_VERSION=latest +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG MAINTAINER="vdaas.org vald team " + +FROM golang:${GO_VERSION} AS golang + +FROM ubuntu:devel AS builder + +ENV GO111MODULE on +ENV DEBIAN_FRONTEND noninteractive +ENV INITRD No +ENV LANG en_US.UTF-8 +ENV GOROOT /opt/go +ENV GOPATH /go +ENV PATH ${PATH}:${GOROOT}/bin:${GOPATH}/bin +ENV ORG vdaas +ENV REPO vald +ENV PKG index/job/creation +ENV APP_NAME index-creation + +# skipcq: DOK-DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + curl \ + upx \ + git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=golang /usr/local/go $GOROOT +RUN mkdir -p "$GOPATH/src" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY Makefile . +COPY .git . +COPY go.mod . +COPY go.sum . + +RUN make go/download + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal +COPY internal . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc +COPY apis/grpc . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} +COPY pkg/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +COPY cmd/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +RUN cp sample.yaml /tmp/config.yaml + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer="${MAINTAINER}" + +ENV APP_NAME index-creation + +COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +COPY --from=builder /tmp/config.yaml /etc/server/config.yaml + +USER nonroot:nonroot + +ENTRYPOINT ["/go/bin/index-creation"] diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile new file mode 100644 index 0000000000..9e7361d93b --- /dev/null +++ b/dockers/index/job/save/Dockerfile @@ -0,0 +1,93 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG GO_VERSION=latest +ARG DISTROLESS_IMAGE=gcr.io/distroless/static +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG MAINTAINER="vdaas.org vald team " + +FROM golang:${GO_VERSION} AS golang + +FROM ubuntu:devel AS builder + +ENV GO111MODULE on +ENV DEBIAN_FRONTEND noninteractive +ENV INITRD No +ENV LANG en_US.UTF-8 +ENV GOROOT /opt/go +ENV GOPATH /go +ENV PATH ${PATH}:${GOROOT}/bin:${GOPATH}/bin +ENV ORG vdaas +ENV REPO vald +ENV PKG index/job/save +ENV APP_NAME index-save + +# skipcq: DOK-DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + curl \ + upx \ + git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=golang /usr/local/go $GOROOT +RUN mkdir -p "$GOPATH/src" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY Makefile . +COPY .git . +COPY go.mod . +COPY go.sum . + +RUN make go/download + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal +COPY internal . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc +COPY apis/grpc . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG} +COPY pkg/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +COPY cmd/${PKG} . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +RUN cp sample.yaml /tmp/config.yaml + +FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer="${MAINTAINER}" + +ENV APP_NAME index-save + +COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME} +COPY --from=builder /tmp/config.yaml /etc/server/config.yaml + +USER nonroot:nonroot + +ENTRYPOINT ["/go/bin/index-save"] diff --git a/docs/tutorial/get-started.md b/docs/tutorial/get-started.md index ec8b8addea..683703cb10 100644 --- a/docs/tutorial/get-started.md +++ b/docs/tutorial/get-started.md @@ -67,6 +67,12 @@ Please make sure these functions are available.
The configuration of Kubernetes Ingress is depended on your Kubernetes cluster's provider. Please refer to on yourself. +In the following example, we create the Kubernetes cluster using [k3d](https://k3d.io/), that the internal port 80 (where the traefik ingress controller is listening on) is exposed on the host system. + +```bash +k3d cluster create -p 8081:80@loadbalancer +``` + The way to deploy Kubernetes Metrics Service is here: ```bash @@ -108,7 +114,7 @@ In this tutorial, you will deploy the basic configuration of Vald that is consis enabled: true # TODO: Set your ingress host. host: localhost - # TODO: Set annotations which you have to set for your k8s cluster. + # TODO: Set annotations which you have to set for your Ingress resource. annotations: ... ``` @@ -164,8 +170,8 @@ In this tutorial, you will deploy the basic configuration of Vald that is consis
Example output
```bash - NAME CLASS HOSTS ADDRESS PORTS AGE - vald-lb-gateway-ingress localhost 192.168.16.2 80 7m43s + NAME CLASS HOSTS ADDRESS PORTS AGE + vald-lb-gateway-ingress traefik localhost 192.168.16.2 80 7m43s ```
@@ -253,9 +259,9 @@ If you are interested, please refer to [SDKs](../user-guides/sdks.md).
"github.com/kpango/glg" "github.com/vdaas/vald-client-go/v1/payload" "github.com/vdaas/vald-client-go/v1/vald" - "gonum.org/v1/hdf5" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) ``` @@ -323,7 +329,7 @@ If you are interested, please refer to [SDKs](../user-guides/sdks.md).
```go ctx := context.Background() - conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { glg.Fatal(err) } diff --git a/docs/tutorial/vald-agent-standalone-on-k8s.md b/docs/tutorial/vald-agent-standalone-on-k8s.md index 7ac4e73265..095b6831e8 100644 --- a/docs/tutorial/vald-agent-standalone-on-k8s.md +++ b/docs/tutorial/vald-agent-standalone-on-k8s.md @@ -165,11 +165,11 @@ This chapter uses [NGT](https://github.com/yahoojapan/ngt) as Vald Agent to perf "github.com/kpango/fuid" "github.com/kpango/glg" agent "github.com/vdaas/vald-client-go/v1/agent/core" - "github.com/vdaas/vald-client-go/v1/vald" "github.com/vdaas/vald-client-go/v1/payload" - + "github.com/vdaas/vald-client-go/v1/vald" "gonum.org/v1/hdf5" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) ``` @@ -236,7 +236,7 @@ This chapter uses [NGT](https://github.com/yahoojapan/ngt) as Vald Agent to perf ```go ctx := context.Background() - conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { glg.Fatal(err) } @@ -284,21 +284,23 @@ This chapter uses [NGT](https://github.com/yahoojapan/ngt) as Vald Agent to perf - - [Optional] Indexing manually instead of waiting for auto indexing + - [Optional] Indexing manually instead of waiting for auto indexing. + You can set Agent NGT configuration `auto_index_duration_limit` and `auto_index_check_duration` for auto indexing. In this example, you can create index manually using `CreateAndSaveIndex()` method in the client library -
example code
- ```go - _, err = client.CreateAndSaveIndex(ctx, &payload.Control_CreateIndexRequest{ - PoolSize: uint32(insertCount), - }) - if err != nil { - glg.Fatal(err) - } - ``` +
example code
-
+ ```go + _, err = client.CreateAndSaveIndex(ctx, &payload.Control_CreateIndexRequest{ + PoolSize: uint32(insertCount), + }) + if err != nil { + glg.Fatal(err) + } + ``` + +
1. Search @@ -333,49 +335,48 @@ This chapter uses [NGT](https://github.com/yahoojapan/ngt) as Vald Agent to perf 1. Remove - - Remove indexed 400 training datasets from the Vald agent. -
example code
- - ```go - for i := range ids [:insertCount] { - _, err := client.Remove(ctx, &payload.Remove_Request{ - Id: &payload.Object_ID{ - Id: ids[i], - }, - }) - if err != nil { - glg.Fatal(err) - } - if i%10 == 0 { - glg.Infof("Removed %d", i) - } - } - ``` - -
- - - - Remove from the index manually instead of waiting for auto indexing. - The removed vectors still exist in the NGT graph index before the SaveIndex (or CreateAndSaveIndex) API is called. - If you run the below code, the indexes will be removed completely from the Vald Agent NGT graph and the Backup file. -
example code
- - ```go - _, err = client.SaveIndex(ctx, &payload.Empty{}) - if err != nil { - glg.Fatal(err) - } - ``` - -
+ - Remove indexed 400 training datasets from the Vald agent. +
example code
-
+ ```go + for i := range ids [:insertCount] { + _, err := client.Remove(ctx, &payload.Remove_Request{ + Id: &payload.Object_ID{ + Id: ids[i], + }, + }) + if err != nil { + glg.Fatal(err) + } + if i%10 == 0 { + glg.Infof("Removed %d", i) + } + } + ``` + + + + - Remove from the index manually instead of waiting for auto indexing. + + The removed vectors still exist in the NGT graph index before the SaveIndex (or CreateAndSaveIndex) API is called. + If you run the below code, the indexes will be removed completely from the Vald Agent NGT graph and the Backup file. + +
example code
+ + ```go + _, err = client.SaveIndex(ctx, &payload.Empty{}) + if err != nil { + glg.Fatal(err) + } + ``` + +
It would be best to run CreateIndex() after Insert() without waiting for auto-indexing in your client code, even you can wait for the finishing auto createIndex function, which sometimes takes a long time. The backup files (e.g., ngt-meta.kvsdb) will be in your mount directory when vald-agent-ngt finishes indexing.
- +
If you use Go(v1.16~) and catch the error like `missing go.sum entry to add it` when running `go run main.go`, please run `go mod tidy` and retry. This error comes from Go Command Changes of Go 1.16 Release Notes.(Please refer to https://golang.org/doc/go1.16#go-command for more details). diff --git a/docs/user-guides/index-correction.md b/docs/user-guides/index-correction.md new file mode 100644 index 0000000000..f6759ab57f --- /dev/null +++ b/docs/user-guides/index-correction.md @@ -0,0 +1,39 @@ +# Index Correction + +In the Vald cluster, the same Index is replicated to multiple agents due to the `index_replica` setting. However, inconsistencies between replicas may occur due to pod eviction or the occurrence of OOM killer during vector insertions. For example, + +1. The timestamp of the index differs between agents (some agents have an old index saved and it has not been updated). +2. The number of replicas does not meet the value set in `index_replica`. + +To resolve these inconsistencies, you can use the `Index Correction` feature. + +`Index Correction` is implemented as a [`CronJob`](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/), checking the consistency between replicas regularly and resolving any inconsistencies. + +## Settings + +- enabled + Turns the index correction feature on/off. +- schedule + Sets the interval for the job start in cron notation (the default value is `3 6 * * *`, which means 3:06 AM every day). +- suspend + [Temporary suspension setting](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-suspension) for CronJob. + +```yaml +manager: + index: + corrector: + enabled: true + schedule: "3 6 * * *" + suspend: false +``` + +## Important Notes + +- Processing time + Under conditions of 10 million identical vectors(not including `index_replica`) and 10 agent replicas, the processing takes about 30~40 minutes (this is only a reference, and the actual execution time may vary depending on the infrastructure). Time complexity of the process is `O(MN)` where M is the number of identical vector items and N is the number of agent replicas. `index_replica` does not matter for the processing time. + +- concurrencyPolicy + `Forbid` is set internally, so a new job will not be created while an existing job is running. In other words, if the process does not finish within the interval specified by the schedule, the next job will not be scheduled. + +- Index operations during correction + Vector operations performed after the start of the index correction job are not considered in that job. diff --git a/example/client/agent/main.go b/example/client/agent/main.go index 493687e337..45cfcc400e 100644 --- a/example/client/agent/main.go +++ b/example/client/agent/main.go @@ -27,6 +27,7 @@ import ( "github.com/vdaas/vald-client-go/v1/vald" "gonum.org/v1/hdf5" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) const ( @@ -65,7 +66,7 @@ func main() { ctx := context.Background() // Create a Vald Agent client for connecting to the Vald cluster. - conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { glg.Fatal(err) } diff --git a/example/client/go.mod b/example/client/go.mod index a8e2c37c62..a356acea23 100644 --- a/example/client/go.mod +++ b/example/client/go.mod @@ -1,20 +1,20 @@ module github.com/vdaas/vald/example/client -go 1.20 +go 1.21 replace ( github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate v1.0.2 github.com/goccy/go-json => github.com/goccy/go-json v0.10.2 github.com/golang/protobuf => github.com/golang/protobuf v1.5.3 github.com/kpango/glg => github.com/kpango/glg v1.6.15 - github.com/pkg/sftp => github.com/pkg/sftp v1.13.5 - golang.org/x/crypto => golang.org/x/crypto v0.10.0 - golang.org/x/net => golang.org/x/net v0.11.0 - golang.org/x/text => golang.org/x/text v0.10.0 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230626202813-9b080da550b3 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3 - google.golang.org/grpc => google.golang.org/grpc v1.56.1 + github.com/pkg/sftp => github.com/pkg/sftp v1.13.6 + golang.org/x/crypto => golang.org/x/crypto v0.14.0 + golang.org/x/net => golang.org/x/net v0.17.0 + golang.org/x/text => golang.org/x/text v0.13.0 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b + google.golang.org/grpc => google.golang.org/grpc v1.58.3 google.golang.org/protobuf => google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1 @@ -23,21 +23,22 @@ replace ( require ( github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 github.com/kpango/glg v1.6.14 - github.com/vdaas/vald-client-go v1.7.6 + github.com/vdaas/vald-client-go v1.7.8 gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.58.3 ) require ( - github.com/envoyproxy/protoc-gen-validate v0.10.1 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/kpango/fastime v1.1.9 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect - google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect - google.golang.org/protobuf v1.30.0 // indirect + github.com/planetscale/vtprotobuf v0.5.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/example/client/go.mod.default b/example/client/go.mod.default index b7a8b698b8..396196b96d 100644 --- a/example/client/go.mod.default +++ b/example/client/go.mod.default @@ -1,6 +1,6 @@ module github.com/vdaas/vald/example/client -go 1.20 +go 1.21 replace ( github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate latest diff --git a/example/client/go.sum b/example/client/go.sum index 9b9963489a..b30397075d 100644 --- a/example/client/go.sum +++ b/example/client/go.sum @@ -1,3 +1,4 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -6,34 +7,47 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84= github.com/kpango/fastime v1.1.9/go.mod h1:vyD7FnUn08zxY4b/QFBZVG+9EWMYsNl+QF0uE46urD4= github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 h1:rxyM+7uaZQ35P9fbixdnld/h4AgEhODoubuy6A4nDdk= github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1/go.mod h1:CAYeq6us9NfnRkSz67/xKVIR6/vaY5ZQZRe6IVcaIKg= github.com/kpango/glg v1.6.15 h1:nw0xSxpSyrDIWHeb3dvnE08PW+SCbK+aYFETT75IeLA= github.com/kpango/glg v1.6.15/go.mod h1:cmsc7Yeu8AS3wHLmN7bhwENXOpxfq+QoqxCIk2FneRk= +github.com/planetscale/vtprotobuf v0.5.0 h1:l8PXm6Colok5z6qQLNhAj2Jq5BfoMTIHxLER5a6nDqM= +github.com/planetscale/vtprotobuf v0.5.0/go.mod h1:wm1N3qk9G/4+VM1WhpkLbvY/d8+0PbwYYpP5P5VhTks= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/vdaas/vald-client-go v1.7.6 h1:rvd7Rt6Xx917r9Yy9sB4wnYCsguegXXaQUME9JquxGU= -github.com/vdaas/vald-client-go v1.7.6/go.mod h1:rJcmCO0yuJEb4vd8jh7aDCgiZr7b10g3SYvfB9wt1O8= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/vdaas/vald-client-go v1.7.8 h1:QguvA8BTj8o3Xe2FfmSO8ipOBTTdXoTB1KnSL52+IvM= +github.com/vdaas/vald-client-go v1.7.8/go.mod h1:kZ4GSvNGrCIaD3HfA7quabWM8Z10roUzT4C4xgCSTqU= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= -google.golang.org/genproto v0.0.0-20230626202813-9b080da550b3 h1:Yofj1/U0xc/Zi5KEpoIxm51I2f85X+eGyY4YzAujRdw= -google.golang.org/genproto v0.0.0-20230626202813-9b080da550b3/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3 h1:wl7z+A0jkB3Rl8Hz74SqGDlnnn5VlL2CV+9UTdZOo00= -google.golang.org/genproto/googleapis/api v0.0.0-20230626202813-9b080da550b3/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3 h1:QJuqz7YzNTyKDspkp2lrzqtq4lf2AhUSpXTsGP5SbLw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230626202813-9b080da550b3/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/example/client/main.go b/example/client/main.go index 8585c66467..386de99917 100644 --- a/example/client/main.go +++ b/example/client/main.go @@ -25,6 +25,7 @@ import ( "github.com/vdaas/vald-client-go/v1/vald" "gonum.org/v1/hdf5" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) const ( @@ -41,11 +42,11 @@ var ( func init() { /** Path option specifies hdf file by path. Default value is `fashion-mnist-784-euclidean.hdf5`. - Addr option specifies grpc server address. Default value is `127.0.0.1:8080`. + Addr option specifies grpc server address. Default value is `127.0.0.1:8081`. Wait option specifies indexing wait time (in seconds). Default value is `60`. **/ flag.StringVar(&datasetPath, "path", "fashion-mnist-784-euclidean.hdf5", "dataset path") - flag.StringVar(&grpcServerAddr, "addr", "localhost:8080", "gRPC server address") + flag.StringVar(&grpcServerAddr, "addr", "localhost:8081", "gRPC server address") flag.UintVar(&indexingWaitSeconds, "wait", 60, "indexing wait seconds") flag.Parse() } @@ -62,7 +63,7 @@ func main() { ctx := context.Background() // Create a Vald client for connecting to the Vald cluster. - conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithInsecure()) + conn, err := grpc.DialContext(ctx, grpcServerAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { glg.Fatal(err) } diff --git a/example/helm/values-with-pyroscope.yaml b/example/helm/values-with-pyroscope.yaml index df63a9a26a..99947ab594 100644 --- a/example/helm/values-with-pyroscope.yaml +++ b/example/helm/values-with-pyroscope.yaml @@ -54,14 +54,14 @@ gateway: cpu: 150m memory: 150Mi ingress: + # if enabled is true, vald-lb-gateway can be connected through Kubernetes ingress from the external network. enabled: true - annotations: - ingress.kubernetes.io/protocol: h2c - ingress.kubernetes.io/ssl-passthrough: "true" - traefik.protocol: h2c - kubernetes.io/ingress.class: traefik host: localhost servicePort: grpc + service: + # NOTE: https://doc.traefik.io/traefik/routing/providers/kubernetes-ingress/#on-service + annotations: + traefik.ingress.kubernetes.io/service.serversscheme: h2c gateway_config: index_replica: 2 diff --git a/example/helm/values.yaml b/example/helm/values.yaml index 76c6c45ade..8d969e02fb 100644 --- a/example/helm/values.yaml +++ b/example/helm/values.yaml @@ -41,15 +41,13 @@ gateway: enabled: false ingress: # if enabled is true, vald-lb-gateway can be connected through Kubernetes ingress from the external network. - enabled: false + enabled: true # TODO: Set your ingress host. host: localhost + service: + # NOTE: https://doc.traefik.io/traefik/routing/providers/kubernetes-ingress/#on-service annotations: - # TODO: Set annotations which you have to set for your k8s cluster. - ingress.kubernetes.io/protocol: "h2c" - ingress.kubernetes.io/ssl-passthrough: "true" - kubernetes.io/ingress.class: "traefik" - traefik.protocol: "h2c" + traefik.ingress.kubernetes.io/service.serversscheme: h2c ## vald-agent settings agent: @@ -64,7 +62,7 @@ agent: memory: 150Mi ngt: # The number of dimensions for feature vector of fashion-mnist dataset. - dimension: 300 + dimension: 784 # We use L2-Norm for distance_type. distance_type: cos # The type of fashion-mnist's feature vectors. diff --git a/go.mod b/go.mod index 664863bbe0..8dee173bf2 100755 --- a/go.mod +++ b/go.mod @@ -4,17 +4,17 @@ go 1.21 replace ( cloud.google.com/go => cloud.google.com/go v0.110.8 - cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.55.0 - cloud.google.com/go/compute => cloud.google.com/go/compute v1.23.0 - cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.14.0 + cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.56.0 + cloud.google.com/go/compute => cloud.google.com/go/compute v1.23.1 + cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.15.0 cloud.google.com/go/firestore => cloud.google.com/go/firestore v1.13.0 - cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.2 - cloud.google.com/go/kms => cloud.google.com/go/kms v1.15.2 - cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.16.0 + cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.3 + cloud.google.com/go/kms => cloud.google.com/go/kms v1.15.3 + cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.16.1 cloud.google.com/go/pubsub => cloud.google.com/go/pubsub v1.33.0 - cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.11.1 + cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.11.2 cloud.google.com/go/storage => cloud.google.com/go/storage v1.33.0 - cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.1 + cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.2 code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20230612151507-41ef4d1f67a4 contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws v0.0.0-20230502192102-15967c811cec contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus v0.4.2 @@ -22,8 +22,8 @@ replace ( git.sr.ht/~sbinet/gg => git.sr.ht/~sbinet/gg v0.5.0 github.com/Azure/azure-amqp-common-go/v3 => github.com/Azure/azure-amqp-common-go/v3 v3.2.3 github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.2 - github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 github.com/Azure/go-amqp => github.com/Azure/go-amqp v1.0.2 github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.1-0.20230905222633-df94ce56f001+incompatible @@ -38,34 +38,34 @@ replace ( github.com/DATA-DOG/go-sqlmock => github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/GoogleCloudPlatform/cloudsql-proxy => github.com/GoogleCloudPlatform/cloudsql-proxy v1.33.11 github.com/Masterminds/semver/v3 => github.com/Masterminds/semver/v3 v3.2.1 - github.com/ajstarks/deck => github.com/ajstarks/deck v0.0.0-20230917174535-ca5324ee72b2 - github.com/ajstarks/deck/generate => github.com/ajstarks/deck/generate v0.0.0-20230917174535-ca5324ee72b2 + github.com/ajstarks/deck => github.com/ajstarks/deck v0.0.0-20231012031509-f833e437b68a + github.com/ajstarks/deck/generate => github.com/ajstarks/deck/generate v0.0.0-20231012031509-f833e437b68a github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b github.com/antihax/optional => github.com/antihax/optional v1.0.0 github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.14 - github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.21.0 - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 - github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.41 - github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.39 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 - github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.85 - github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 - github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 - github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 - github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 - github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms v1.24.5 - github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 v1.39.0 - github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.3 - github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.22.0 - github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.24.5 - github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.37.5 - github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 - github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 - github.com/aws/smithy-go => github.com/aws/smithy-go v1.14.2 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.26 + github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.21.2 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 + github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.19.0 + github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.43 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 + github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91 + github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 + github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 + github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 + github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms v1.24.7 + github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 + github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.5 + github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.22.2 + github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.24.7 + github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.38.2 + github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 + github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 + github.com/aws/smithy-go => github.com/aws/smithy-go v1.15.0 github.com/benbjohnson/clock => github.com/benbjohnson/clock v1.3.5 github.com/beorn7/perks => github.com/beorn7/perks v1.0.1 github.com/bmizerany/assert => github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 @@ -78,10 +78,10 @@ replace ( github.com/chzyer/readline => github.com/chzyer/readline v1.5.1 github.com/chzyer/test => github.com/chzyer/test v1.0.0 github.com/cncf/udpa/go => github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe - github.com/cncf/xds/go => github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 + github.com/cncf/xds/go => github.com/cncf/xds/go v0.0.0-20231016030527-8bd2eac9fb4a github.com/cockroachdb/apd => github.com/cockroachdb/apd v1.1.0 github.com/coreos/go-systemd/v22 => github.com/coreos/go-systemd/v22 v22.5.0 - github.com/cpuguy83/go-md2man/v2 => github.com/cpuguy83/go-md2man/v2 v2.0.2 + github.com/cpuguy83/go-md2man/v2 => github.com/cpuguy83/go-md2man/v2 v2.0.3 github.com/creack/pty => github.com/creack/pty v1.1.18 github.com/davecgh/go-spew => github.com/davecgh/go-spew v1.1.1 github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb v0.12.3 @@ -120,7 +120,7 @@ replace ( github.com/go-playground/assert/v2 => github.com/go-playground/assert/v2 v2.2.0 github.com/go-playground/locales => github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator => github.com/go-playground/universal-translator v0.18.1 - github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.15.4 + github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.15.5 github.com/go-redis/redis/v8 => github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.7.1 github.com/go-task/slim-sprig => github.com/go-task/slim-sprig v2.20.0+incompatible @@ -132,7 +132,7 @@ replace ( github.com/gobwas/ws => github.com/gobwas/ws v1.3.0 github.com/goccy/go-json => github.com/goccy/go-json v0.10.2 github.com/gocql/gocql => github.com/gocql/gocql v1.6.0 - github.com/gocraft/dbr/v2 => github.com/gocraft/dbr/v2 v2.7.5 + github.com/gocraft/dbr/v2 => github.com/gocraft/dbr/v2 v2.7.6 github.com/godbus/dbus/v5 => github.com/godbus/dbus/v5 v5.1.0 github.com/gofrs/uuid => github.com/gofrs/uuid v4.4.0+incompatible github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2 @@ -146,14 +146,14 @@ replace ( github.com/golang/protobuf => github.com/golang/protobuf v1.5.3 github.com/golang/snappy => github.com/golang/snappy v0.0.4 github.com/google/btree => github.com/google/btree v1.1.2 - github.com/google/gnostic => github.com/google/gnostic v0.6.9 - github.com/google/go-cmp => github.com/google/go-cmp v0.5.9 + github.com/google/gnostic => github.com/google/gnostic v0.7.0 + github.com/google/go-cmp => github.com/google/go-cmp v0.6.0 github.com/google/go-replayers/grpcreplay => github.com/google/go-replayers/grpcreplay v1.1.0 github.com/google/go-replayers/httpreplay => github.com/google/go-replayers/httpreplay v1.2.0 github.com/google/gofuzz => github.com/google/gofuzz v1.2.0 github.com/google/martian => github.com/google/martian v2.1.0+incompatible github.com/google/martian/v3 => github.com/google/martian/v3 v3.3.2 - github.com/google/pprof => github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8 + github.com/google/pprof => github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 github.com/google/shlex => github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/subcommands => github.com/google/subcommands v1.2.0 github.com/google/uuid => github.com/google/uuid v1.3.1 @@ -190,7 +190,7 @@ replace ( github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v1.0.0 github.com/kisielk/errcheck => github.com/kisielk/errcheck v1.6.3 github.com/kisielk/gotool => github.com/kisielk/gotool v1.0.0 - github.com/klauspost/compress => github.com/klauspost/compress v1.17.1-0.20230918170125-6125c11bae77 + github.com/klauspost/compress => github.com/klauspost/compress v1.17.1 github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.5 github.com/kpango/fastime => github.com/kpango/fastime v1.1.9 github.com/kpango/fuid => github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 @@ -220,10 +220,10 @@ replace ( github.com/montanaflynn/stats => github.com/montanaflynn/stats v0.7.1 github.com/munnerz/goautoneg => github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 github.com/niemeyer/pretty => github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e - github.com/nxadm/tail => github.com/nxadm/tail v1.4.8 + github.com/nxadm/tail => github.com/nxadm/tail v1.4.11 github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.16.5 - github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 v2.12.1 - github.com/onsi/gomega => github.com/onsi/gomega v1.27.10 + github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 v2.13.0 + github.com/onsi/gomega => github.com/onsi/gomega v1.28.0 github.com/peterbourgon/diskv => github.com/peterbourgon/diskv v2.0.1+incompatible github.com/phpdave11/gofpdf => github.com/phpdave11/gofpdf v1.4.2 github.com/phpdave11/gofpdi => github.com/phpdave11/gofpdi v1.0.13 @@ -234,10 +234,10 @@ replace ( github.com/pkg/sftp => github.com/pkg/sftp v1.13.6 github.com/pmezard/go-difflib => github.com/pmezard/go-difflib v1.0.0 github.com/prashantv/gostub => github.com/prashantv/gostub v1.1.0 - github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.16.0 - github.com/prometheus/client_model => github.com/prometheus/client_model v0.4.0 + github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_model => github.com/prometheus/client_model v0.5.0 github.com/prometheus/common => github.com/prometheus/common v0.44.0 - github.com/prometheus/procfs => github.com/prometheus/procfs v0.11.1 + github.com/prometheus/procfs => github.com/prometheus/procfs v0.12.0 github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.99.0 github.com/quasilyte/go-ruleguard => github.com/quasilyte/go-ruleguard v0.4.0 github.com/quasilyte/go-ruleguard/dsl => github.com/quasilyte/go-ruleguard/dsl v0.3.22 @@ -246,7 +246,7 @@ replace ( github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid v1.2.0 github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal v1.11.0 github.com/rs/xid => github.com/rs/xid v1.5.0 - github.com/rs/zerolog => github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog => github.com/rs/zerolog v1.31.0 github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 v2.1.0 github.com/ruudk/golang-pdf417 => github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245 github.com/schollz/progressbar/v2 => github.com/schollz/progressbar/v2 v2.15.0 @@ -257,7 +257,7 @@ replace ( github.com/shurcooL/httpfs => github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c github.com/shurcooL/vfsgen => github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 github.com/sirupsen/logrus => github.com/sirupsen/logrus v1.9.3 - github.com/spf13/afero => github.com/spf13/afero v1.9.5 + github.com/spf13/afero => github.com/spf13/afero v1.10.0 github.com/spf13/cobra => github.com/spf13/cobra v1.7.0 github.com/spf13/pflag => github.com/spf13/pflag v1.0.5 github.com/stoewer/go-strcase => github.com/stoewer/go-strcase v1.3.0 @@ -284,39 +284,39 @@ replace ( go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.33.0 go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1 go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v1.0.0 - go.starlark.net => go.starlark.net v0.0.0-20230912135651-745481cf39ed + go.starlark.net => go.starlark.net v0.0.0-20231016134836-22325403fcb3 go.uber.org/atomic => go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs => go.uber.org/automaxprocs v1.5.3 go.uber.org/goleak => go.uber.org/goleak v1.2.1 go.uber.org/multierr => go.uber.org/multierr v1.11.0 go.uber.org/zap => go.uber.org/zap v1.26.0 gocloud.dev => gocloud.dev v0.34.0 - golang.org/x/crypto => golang.org/x/crypto v0.13.0 - golang.org/x/exp => golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 - golang.org/x/image => golang.org/x/image v0.12.0 + golang.org/x/crypto => golang.org/x/crypto v0.14.0 + golang.org/x/exp => golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20231006140011-7918f672742d + golang.org/x/image => golang.org/x/image v0.13.0 golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 - golang.org/x/mobile => golang.org/x/mobile v0.0.0-20230906132913-2077a3224571 - golang.org/x/mod => golang.org/x/mod v0.12.0 - golang.org/x/net => golang.org/x/net v0.15.0 - golang.org/x/oauth2 => golang.org/x/oauth2 v0.12.0 - golang.org/x/sync => golang.org/x/sync v0.3.0 - golang.org/x/sys => golang.org/x/sys v0.12.0 - golang.org/x/term => golang.org/x/term v0.12.0 + golang.org/x/mobile => golang.org/x/mobile v0.0.0-20231006135142-2b44d11868fe + golang.org/x/mod => golang.org/x/mod v0.13.0 + golang.org/x/net => golang.org/x/net v0.17.0 + golang.org/x/oauth2 => golang.org/x/oauth2 v0.13.0 + golang.org/x/sync => golang.org/x/sync v0.4.0 + golang.org/x/sys => golang.org/x/sys v0.13.0 + golang.org/x/term => golang.org/x/term v0.13.0 golang.org/x/text => golang.org/x/text v0.13.0 golang.org/x/time => golang.org/x/time v0.3.0 - golang.org/x/tools => golang.org/x/tools v0.13.0 - golang.org/x/xerrors => golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 + golang.org/x/tools => golang.org/x/tools v0.14.0 + golang.org/x/xerrors => golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 gomodules.xyz/jsonpatch/v2 => gomodules.xyz/jsonpatch/v2 v2.4.0 gonum.org/v1/gonum => gonum.org/v1/gonum v0.14.0 gonum.org/v1/hdf5 => gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 gonum.org/v1/plot => gonum.org/v1/plot v0.14.0 - google.golang.org/api => google.golang.org/api v0.142.0 + google.golang.org/api => google.golang.org/api v0.147.0 google.golang.org/appengine => google.golang.org/appengine v1.6.8 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 - google.golang.org/grpc => google.golang.org/grpc v1.58.1 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b + google.golang.org/grpc => google.golang.org/grpc v1.58.3 google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf => google.golang.org/protobuf v1.31.0 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c @@ -388,17 +388,17 @@ require ( go.uber.org/goleak v1.2.1 go.uber.org/zap v1.24.0 gocloud.dev v0.0.0-00010101000000-000000000000 - golang.org/x/net v0.15.0 - golang.org/x/oauth2 v0.12.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.12.0 + golang.org/x/net v0.17.0 + golang.org/x/oauth2 v0.13.0 + golang.org/x/sync v0.4.0 + golang.org/x/sys v0.13.0 golang.org/x/text v0.13.0 - golang.org/x/tools v0.13.0 + golang.org/x/tools v0.14.0 gonum.org/v1/hdf5 v0.0.0-00010101000000-000000000000 gonum.org/v1/plot v0.10.1 - google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb - google.golang.org/genproto/googleapis/rpc v0.0.0-20230920183334-c177e329c48b - google.golang.org/grpc v1.58.0 + google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a + google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a + google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.27.3 @@ -411,9 +411,9 @@ require ( require ( cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.2 // indirect + cloud.google.com/go/iam v1.1.3 // indirect git.sr.ht/~sbinet/gg v0.5.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect @@ -446,11 +446,12 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.5.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect @@ -474,9 +475,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.15.1 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/scylladb/go-reflectx v1.0.1 // indirect @@ -490,17 +491,17 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.13.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833 // indirect - golang.org/x/image v0.12.0 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/image v0.13.0 // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect - google.golang.org/api v0.134.0 // indirect + google.golang.org/api v0.139.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect + google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.2 // indirect diff --git a/go.sum b/go.sum index 69e26ece2d..de5cead015 100644 --- a/go.sum +++ b/go.sum @@ -1,137 +1,138 @@ cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= -cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= +cloud.google.com/go/accessapproval v1.7.2/go.mod h1:/gShiq9/kK/h8T/eEn1BTzalDvk0mZxJlhfw0p+Xuc0= cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= -cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= -cloud.google.com/go/aiplatform v1.50.0/go.mod h1:IRc2b8XAMTa9ZmfJV1BCCQbieWWvDnP1A8znyz5N7y4= -cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= -cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= -cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= -cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= -cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= -cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= -cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= -cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= -cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= -cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= -cloud.google.com/go/baremetalsolution v1.2.0/go.mod h1:68wi9AwPYkEWIUT4SvSGS9UJwKzNpshjHsH4lzk8iOw= -cloud.google.com/go/batch v1.4.1/go.mod h1:KdBmDD61K0ovcxoRHGrN6GmOBWeAOyCgKD0Mugx4Fkk= -cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= -cloud.google.com/go/bigquery v1.55.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec= -cloud.google.com/go/billing v1.17.0/go.mod h1:Z9+vZXEq+HwH7bhJkyI4OQcR6TSbeMrjlpEjO2vzY64= -cloud.google.com/go/binaryauthorization v1.7.0/go.mod h1:Zn+S6QqTMn6odcMU1zDZCJxPjU2tZPV1oDl45lWY154= -cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= -cloud.google.com/go/channel v1.17.0/go.mod h1:RpbhJsGi/lXWAUM1eF4IbQGbsfVlg2o8Iiy2/YLfVT0= -cloud.google.com/go/cloudbuild v1.14.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= -cloud.google.com/go/clouddms v1.7.0/go.mod h1:MW1dC6SOtI/tPNCciTsXtsGNEM0i0OccykPvv3hiYeM= -cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/accesscontextmanager v1.8.2/go.mod h1:E6/SCRM30elQJ2PKtFMs2YhfJpZSNcJyejhuzoId4Zk= +cloud.google.com/go/aiplatform v1.51.1/go.mod h1:kY3nIMAVQOK2XDqDPHaOuD9e+FdMA6OOpfBjsvaFSOo= +cloud.google.com/go/analytics v0.21.4/go.mod h1:zZgNCxLCy8b2rKKVfC1YkC2vTrpfZmeRCySM3aUbskA= +cloud.google.com/go/apigateway v1.6.2/go.mod h1:CwMC90nnZElorCW63P2pAYm25AtQrHfuOkbRSHj0bT8= +cloud.google.com/go/apigeeconnect v1.6.2/go.mod h1:s6O0CgXT9RgAxlq3DLXvG8riw8PYYbU/v25jqP3Dy18= +cloud.google.com/go/apigeeregistry v0.7.2/go.mod h1:9CA2B2+TGsPKtfi3F7/1ncCCsL62NXBRfM6iPoGSM+8= +cloud.google.com/go/appengine v1.8.2/go.mod h1:WMeJV9oZ51pvclqFN2PqHoGnys7rK0rz6s3Mp6yMvDo= +cloud.google.com/go/area120 v0.8.2/go.mod h1:a5qfo+x77SRLXnCynFWPUZhnZGeSgvQ+Y0v1kSItkh4= +cloud.google.com/go/artifactregistry v1.14.3/go.mod h1:A2/E9GXnsyXl7GUvQ/2CjHA+mVRoWAXC0brg2os+kNI= +cloud.google.com/go/asset v1.15.1/go.mod h1:yX/amTvFWRpp5rcFq6XbCxzKT8RJUam1UoboE179jU4= +cloud.google.com/go/assuredworkloads v1.11.2/go.mod h1:O1dfr+oZJMlE6mw0Bp0P1KZSlj5SghMBvTpZqIcUAW4= +cloud.google.com/go/automl v1.13.2/go.mod h1:gNY/fUmDEN40sP8amAX3MaXkxcqPIn7F1UIIPZpy4Mg= +cloud.google.com/go/baremetalsolution v1.2.1/go.mod h1:3qKpKIw12RPXStwQXcbhfxVj1dqQGEvcmA+SX/mUR88= +cloud.google.com/go/batch v1.5.1/go.mod h1:RpBuIYLkQu8+CWDk3dFD/t/jOCGuUpkpX+Y0n1Xccs8= +cloud.google.com/go/beyondcorp v1.0.1/go.mod h1:zl/rWWAFVeV+kx+X2Javly7o1EIQThU4WlkynffL/lk= +cloud.google.com/go/bigquery v1.56.0/go.mod h1:KDcsploXTEY7XT3fDQzMUZlpQLHzE4itubHrnmhUrZA= +cloud.google.com/go/billing v1.17.2/go.mod h1:u/AdV/3wr3xoRBk5xvUzYMS1IawOAPwQMuHgHMdljDg= +cloud.google.com/go/binaryauthorization v1.7.1/go.mod h1:GTAyfRWYgcbsP3NJogpV3yeunbUIjx2T9xVeYovtURE= +cloud.google.com/go/certificatemanager v1.7.2/go.mod h1:15SYTDQMd00kdoW0+XY5d9e+JbOPjp24AvF48D8BbcQ= +cloud.google.com/go/channel v1.17.1/go.mod h1:xqfzcOZAcP4b/hUDH0GkGg1Sd5to6di1HOJn/pi5uBQ= +cloud.google.com/go/cloudbuild v1.14.1/go.mod h1:K7wGc/3zfvmYWOWwYTgF/d/UVJhS4pu+HAy7PL7mCsU= +cloud.google.com/go/clouddms v1.7.1/go.mod h1:o4SR8U95+P7gZ/TX+YbJxehOCsM+fe6/brlrFquiszk= +cloud.google.com/go/cloudtasks v1.12.2/go.mod h1:A7nYkjNlW2gUoROg1kvJrQGhJP/38UaWwsnuBDOBVUk= +cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= +cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= -cloud.google.com/go/container v1.26.0/go.mod h1:YJCmRet6+6jnYYRS000T6k0D0xUXQgBSaJ7VwI8FBj4= -cloud.google.com/go/containeranalysis v0.11.0/go.mod h1:4n2e99ZwpGxpNcz+YsFT1dfOHPQFGcAC8FN2M2/ne/U= -cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= -cloud.google.com/go/datacatalog v1.17.1/go.mod h1:nCSYFHgtxh2MiEktWIz71s/X+7ds/UT9kp0PC7waCzE= -cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= -cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= -cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= -cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= -cloud.google.com/go/dataplex v1.9.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/contactcenterinsights v1.11.1/go.mod h1:FeNP3Kg8iteKM80lMwSk3zZZKVxr+PGnAId6soKuXwE= +cloud.google.com/go/container v1.26.1/go.mod h1:5smONjPRUxeEpDG7bMKWfDL4sauswqEtnBK1/KKpR04= +cloud.google.com/go/containeranalysis v0.11.1/go.mod h1:rYlUOM7nem1OJMKwE1SadufX0JP3wnXj844EtZAwWLY= +cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.18.1/go.mod h1:TzAWaz+ON1tkNr4MOcak8EBHX7wIRX/gZKM+yTVsv+A= +cloud.google.com/go/dataflow v0.9.2/go.mod h1:vBfdBZ/ejlTaYIGB3zB4T08UshH70vbtZeMD+urnUSo= +cloud.google.com/go/dataform v0.8.2/go.mod h1:X9RIqDs6NbGPLR80tnYoPNiO1w0wenKTb8PxxlhTMKM= +cloud.google.com/go/datafusion v1.7.2/go.mod h1:62K2NEC6DRlpNmI43WHMWf9Vg/YvN6QVi8EVwifElI0= +cloud.google.com/go/datalabeling v0.8.2/go.mod h1:cyDvGHuJWu9U/cLDA7d8sb9a0tWLEletStu2sTmg3BE= +cloud.google.com/go/dataplex v1.10.1/go.mod h1:1MzmBv8FvjYfc7vDdxhnLFNskikkB+3vl475/XdCDhs= cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= -cloud.google.com/go/dataproc/v2 v2.2.0/go.mod h1:lZR7AQtwZPvmINx5J87DSOOpTfof9LVZju6/Qo4lmcY= -cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= -cloud.google.com/go/datastore v1.14.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= -cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= -cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= -cloud.google.com/go/dialogflow v1.43.0/go.mod h1:pDUJdi4elL0MFmt1REMvFkdsUTYSHq+rTCS8wg0S3+M= -cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= -cloud.google.com/go/documentai v1.22.1/go.mod h1:LKs22aDHbJv7ufXuPypzRO7rG3ALLJxzdCXDPutw4Qc= -cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= -cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= +cloud.google.com/go/dataproc/v2 v2.2.1/go.mod h1:QdAJLaBjh+l4PVlVZcmrmhGccosY/omC1qwfQ61Zv/o= +cloud.google.com/go/dataqna v0.8.2/go.mod h1:KNEqgx8TTmUipnQsScOoDpq/VlXVptUqVMZnt30WAPs= +cloud.google.com/go/datastore v1.15.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8= +cloud.google.com/go/datastream v1.10.1/go.mod h1:7ngSYwnw95YFyTd5tOGBxHlOZiL+OtpjheqU7t2/s/c= +cloud.google.com/go/deploy v1.13.1/go.mod h1:8jeadyLkH9qu9xgO3hVWw8jVr29N1mnW42gRJT8GY6g= +cloud.google.com/go/dialogflow v1.44.1/go.mod h1:n/h+/N2ouKOO+rbe/ZnI186xImpqvCVj2DdsWS/0EAk= +cloud.google.com/go/dlp v1.10.2/go.mod h1:ZbdKIhcnyhILgccwVDzkwqybthh7+MplGC3kZVZsIOQ= +cloud.google.com/go/documentai v1.23.2/go.mod h1:Q/wcRT+qnuXOpjAkvOV4A+IeQl04q2/ReT7SSbytLSo= +cloud.google.com/go/domains v0.9.2/go.mod h1:3YvXGYzZG1Temjbk7EyGCuGGiXHJwVNmwIf+E/cUp5I= +cloud.google.com/go/edgecontainer v1.1.2/go.mod h1:wQRjIzqxEs9e9wrtle4hQPSR1Y51kqN75dgF7UllZZ4= cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= -cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= -cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= -cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= +cloud.google.com/go/essentialcontacts v1.6.3/go.mod h1:yiPCD7f2TkP82oJEFXFTou8Jl8L6LBRPeBEkTaO0Ggo= +cloud.google.com/go/eventarc v1.13.1/go.mod h1:EqBxmGHFrruIara4FUQ3RHlgfCn7yo1HYsu2Hpt/C3Y= +cloud.google.com/go/filestore v1.7.2/go.mod h1:TYOlyJs25f/omgj+vY7/tIG/E7BX369triSPzE4LdgE= cloud.google.com/go/firestore v1.13.0/go.mod h1:QojqqOh8IntInDUSTAh0c8ZsPYAr68Ma8c5DWOy8xb8= -cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= -cloud.google.com/go/gkebackup v1.3.1/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= -cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= -cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= -cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/functions v1.15.2/go.mod h1:CHAjtcR6OU4XF2HuiVeriEdELNcnvRZSk1Q8RMqy4lE= +cloud.google.com/go/gkebackup v1.3.2/go.mod h1:OMZbXzEJloyXMC7gqdSB+EOEQ1AKcpGYvO3s1ec5ixk= +cloud.google.com/go/gkeconnect v0.8.2/go.mod h1:6nAVhwchBJYgQCXD2pHBFQNiJNyAd/wyxljpaa6ZPrY= +cloud.google.com/go/gkehub v0.14.2/go.mod h1:iyjYH23XzAxSdhrbmfoQdePnlMj2EWcvnR+tHdBQsCY= +cloud.google.com/go/gkemulticloud v1.0.1/go.mod h1:AcrGoin6VLKT/fwZEYuqvVominLriQBCKmbjtnbMjG8= cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= -cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= -cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= -cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/iap v1.9.0/go.mod h1:01OFxd1R+NFrg78S+hoPV5PxEzv22HXaNqUUlmNHFuY= -cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= -cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= -cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w= -cloud.google.com/go/language v1.11.0/go.mod h1:uDx+pFDdAKTY8ehpWbiXyQdz8tDSYLJbQcXsCkjYyvQ= -cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= +cloud.google.com/go/gsuiteaddons v1.6.2/go.mod h1:K65m9XSgs8hTF3X9nNTPi8IQueljSdYo9F+Mi+s4MyU= +cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= +cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= +cloud.google.com/go/iap v1.9.1/go.mod h1:SIAkY7cGMLohLSdBR25BuIxO+I4fXJiL06IBL7cy/5Q= +cloud.google.com/go/ids v1.4.2/go.mod h1:3vw8DX6YddRu9BncxuzMyWn0g8+ooUjI2gslJ7FH3vk= +cloud.google.com/go/iot v1.7.2/go.mod h1:q+0P5zr1wRFpw7/MOgDXrG/HVA+l+cSwdObffkrpnSg= +cloud.google.com/go/kms v1.15.3/go.mod h1:AJdXqHxS2GlPyduM99s9iGqi2nwbviBbhV/hdmt4iOQ= +cloud.google.com/go/language v1.11.1/go.mod h1:Xyid9MG9WOX3utvDbpX7j3tXDmmDooMyMDqgUVpH17U= +cloud.google.com/go/lifesciences v0.9.2/go.mod h1:QHEOO4tDzcSAzeJg7s2qwnLM2ji8IRpQl4p6m5Z9yTA= cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= -cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= -cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= -cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= -cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= -cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= -cloud.google.com/go/monitoring v1.16.0/go.mod h1:Ptp15HgAyM1fNICAojDMoNc/wUmn67mLHQfyqbw+poY= -cloud.google.com/go/networkconnectivity v1.13.0/go.mod h1:SAnGPes88pl7QRLUen2HmcBSE9AowVAcdug8c0RSBFk= -cloud.google.com/go/networkmanagement v1.9.0/go.mod h1:UTUaEU9YwbCAhhz3jEOHr+2/K/MrBk2XxOLS89LQzFw= -cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= -cloud.google.com/go/notebooks v1.10.0/go.mod h1:SOPYMZnttHxqot0SGSFSkRrwE29eqnKPBJFqgWmiK2k= -cloud.google.com/go/optimization v1.5.0/go.mod h1:evo1OvTxeBRBu6ydPlrIRizKY/LJKo/drDMMRKqGEUU= -cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= +cloud.google.com/go/longrunning v0.5.2/go.mod h1:nqo6DQbNV2pXhGDbDMoN2bWz68MjZUzqv2YttZiveCs= +cloud.google.com/go/managedidentities v1.6.2/go.mod h1:5c2VG66eCa0WIq6IylRk3TBW83l161zkFvCj28X7jn8= +cloud.google.com/go/maps v1.4.1/go.mod h1:BxSa0BnW1g2U2gNdbq5zikLlHUuHW0GFWh7sgML2kIY= +cloud.google.com/go/mediatranslation v0.8.2/go.mod h1:c9pUaDRLkgHRx3irYE5ZC8tfXGrMYwNZdmDqKMSfFp8= +cloud.google.com/go/memcache v1.10.2/go.mod h1:f9ZzJHLBrmd4BkguIAa/l/Vle6uTHzHokdnzSWOdQ6A= +cloud.google.com/go/metastore v1.13.1/go.mod h1:IbF62JLxuZmhItCppcIfzBBfUFq0DIB9HPDoLgWrVOU= +cloud.google.com/go/monitoring v1.16.1/go.mod h1:6HsxddR+3y9j+o/cMJH6q/KJ/CBTvM/38L/1m7bTRJ4= +cloud.google.com/go/networkconnectivity v1.14.1/go.mod h1:LyGPXR742uQcDxZ/wv4EI0Vu5N6NKJ77ZYVnDe69Zug= +cloud.google.com/go/networkmanagement v1.9.1/go.mod h1:CCSYgrQQvW73EJawO2QamemYcOb57LvrDdDU51F0mcI= +cloud.google.com/go/networksecurity v0.9.2/go.mod h1:jG0SeAttWzPMUILEHDUvFYdQTl8L/E/KC8iZDj85lEI= +cloud.google.com/go/notebooks v1.10.1/go.mod h1:5PdJc2SgAybE76kFQCWrTfJolCOUQXF97e+gteUUA6A= +cloud.google.com/go/optimization v1.5.1/go.mod h1:NC0gnUD5MWVAF7XLdoYVPmYYVth93Q6BUzqAq3ZwtV8= +cloud.google.com/go/orchestration v1.8.2/go.mod h1:T1cP+6WyTmh6LSZzeUhvGf0uZVmJyTx7t8z7Vg87+A0= cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= -cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= +cloud.google.com/go/orgpolicy v1.11.2/go.mod h1:biRDpNwfyytYnmCRWZWxrKF22Nkz9eNVj9zyaBdpm1o= cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= -cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= -cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= -cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= -cloud.google.com/go/policytroubleshooter v1.9.0/go.mod h1:+E2Lga7TycpeSTj2FsH4oXxTnrbHJGRlKhVZBLGgU64= -cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= +cloud.google.com/go/osconfig v1.12.2/go.mod h1:eh9GPaMZpI6mEJEuhEjUJmaxvQ3gav+fFEJon1Y8Iw0= +cloud.google.com/go/oslogin v1.11.1/go.mod h1:OhD2icArCVNUxKqtK0mcSmKL7lgr0LVlQz+v9s1ujTg= +cloud.google.com/go/phishingprotection v0.8.2/go.mod h1:LhJ91uyVHEYKSKcMGhOa14zMMWfbEdxG032oT6ECbC8= +cloud.google.com/go/policytroubleshooter v1.9.1/go.mod h1:MYI8i0bCrL8cW+VHN1PoiBTyNZTstCg2WUw2eVC4c4U= +cloud.google.com/go/privatecatalog v0.9.2/go.mod h1:RMA4ATa8IXfzvjrhhK8J6H4wwcztab+oZph3c6WmtFc= cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= -cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= -cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= -cloud.google.com/go/recommender v1.11.0/go.mod h1:kPiRQhPyTJ9kyXPCG6u/dlPLbYfFlkwHNRwdzPVAoII= -cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= -cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= -cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= -cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= -cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= -cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= -cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= -cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= -cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= -cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= -cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= -cloud.google.com/go/spanner v1.49.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM= -cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/recaptchaenterprise/v2 v2.8.1/go.mod h1:JZYZJOeZjgSSTGP4uz7NlQ4/d1w5hGmksVgM0lbEij0= +cloud.google.com/go/recommendationengine v0.8.2/go.mod h1:QIybYHPK58qir9CV2ix/re/M//Ty10OxjnnhWdaKS1Y= +cloud.google.com/go/recommender v1.11.1/go.mod h1:sGwFFAyI57v2Hc5LbIj+lTwXipGu9NW015rkaEM5B18= +cloud.google.com/go/redis v1.13.2/go.mod h1:0Hg7pCMXS9uz02q+LoEVl5dNHUkIQv+C/3L76fandSA= +cloud.google.com/go/resourcemanager v1.9.2/go.mod h1:OujkBg1UZg5lX2yIyMo5Vz9O5hf7XQOSV7WxqxxMtQE= +cloud.google.com/go/resourcesettings v1.6.2/go.mod h1:mJIEDd9MobzunWMeniaMp6tzg4I2GvD3TTmPkc8vBXk= +cloud.google.com/go/retail v1.14.2/go.mod h1:W7rrNRChAEChX336QF7bnMxbsjugcOCPU44i5kbLiL8= +cloud.google.com/go/run v1.3.1/go.mod h1:cymddtZOzdwLIAsmS6s+Asl4JoXIDm/K1cpZTxV4Q5s= +cloud.google.com/go/scheduler v1.10.2/go.mod h1:O3jX6HRH5eKCA3FutMw375XHZJudNIKVonSCHv7ropY= +cloud.google.com/go/secretmanager v1.11.2/go.mod h1:MQm4t3deoSub7+WNwiC4/tRYgDBHJgJPvswqQVB1Vss= +cloud.google.com/go/security v1.15.2/go.mod h1:2GVE/v1oixIRHDaClVbHuPcZwAqFM28mXuAKCfMgYIg= +cloud.google.com/go/securitycenter v1.23.1/go.mod h1:w2HV3Mv/yKhbXKwOCu2i8bCuLtNP1IMHuiYQn4HJq5s= +cloud.google.com/go/servicedirectory v1.11.1/go.mod h1:tJywXimEWzNzw9FvtNjsQxxJ3/41jseeILgwU/QLrGI= +cloud.google.com/go/shell v1.7.2/go.mod h1:KqRPKwBV0UyLickMn0+BY1qIyE98kKyI216sH/TuHmc= +cloud.google.com/go/spanner v1.50.0/go.mod h1:eGj9mQGK8+hkgSVbHNQ06pQ4oS+cyc4tXXd6Dif1KoM= +cloud.google.com/go/speech v1.19.1/go.mod h1:WcuaWz/3hOlzPFOVo9DUsblMIHwxP589y6ZMtaG+iAA= cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M= cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= -cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= -cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= -cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= -cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= -cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= -cloud.google.com/go/translate v1.9.0/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= -cloud.google.com/go/video v1.20.0/go.mod h1:U3G3FTnsvAGqglq9LxgqzOiBc/Nt8zis8S+850N2DUM= -cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= -cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= -cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= -cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= -cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= -cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= -cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= -cloud.google.com/go/workflows v1.12.0/go.mod h1:PYhSk2b6DhZ508tj8HXKaBh+OFe+xdl0dHF/tJdzPQM= +cloud.google.com/go/storagetransfer v1.10.1/go.mod h1:rS7Sy0BtPviWYTTJVWCSV4QrbBitgPeuK4/FKa4IdLs= +cloud.google.com/go/talent v1.6.3/go.mod h1:xoDO97Qd4AK43rGjJvyBHMskiEf3KulgYzcH6YWOVoo= +cloud.google.com/go/texttospeech v1.7.2/go.mod h1:VYPT6aTOEl3herQjFHYErTlSZJ4vB00Q2ZTmuVgluD4= +cloud.google.com/go/tpu v1.6.2/go.mod h1:NXh3NDwt71TsPZdtGWgAG5ThDfGd32X1mJ2cMaRlVgU= +cloud.google.com/go/trace v1.10.2/go.mod h1:NPXemMi6MToRFcSxRl2uDnu/qAlAQ3oULUphcHGh1vA= +cloud.google.com/go/translate v1.9.1/go.mod h1:TWIgDZknq2+JD4iRcojgeDtqGEp154HN/uL6hMvylS8= +cloud.google.com/go/video v1.20.1/go.mod h1:3gJS+iDprnj8SY6pe0SwLeC5BUW80NjhwX7INWEuWGU= +cloud.google.com/go/videointelligence v1.11.2/go.mod h1:ocfIGYtIVmIcWk1DsSGOoDiXca4vaZQII1C85qtoplc= +cloud.google.com/go/vision/v2 v2.7.3/go.mod h1:V0IcLCY7W+hpMKXK1JYE0LV5llEqVmj+UJChjvA1WsM= +cloud.google.com/go/vmmigration v1.7.2/go.mod h1:iA2hVj22sm2LLYXGPT1pB63mXHhrH1m/ruux9TwWLd8= +cloud.google.com/go/vmwareengine v1.0.1/go.mod h1:aT3Xsm5sNx0QShk1Jc1B8OddrxAScYLwzVoaiXfdzzk= +cloud.google.com/go/vpcaccess v1.7.2/go.mod h1:mmg/MnRHv+3e8FJUjeSibVFvQF1cCy2MsFaFqxeY1HU= +cloud.google.com/go/webrisk v1.9.2/go.mod h1:pY9kfDgAqxUpDBOrG4w8deLfhvJmejKB0qd/5uQIPBc= +cloud.google.com/go/websecurityscanner v1.6.2/go.mod h1:7YgjuU5tun7Eg2kpKgGnDuEOXWIrh8x8lWrJT4zfmas= +cloud.google.com/go/workflows v1.12.1/go.mod h1:5A95OhD/edtOhQd/O741NSfIMezNTbCwLM1P1tBRGHM= code.cloudfoundry.org/bytefmt v0.0.0-20230612151507-41ef4d1f67a4 h1:9G5F8zgma5v0GdDvNz6iZwwJp3RS/z0SY/aHGfVwvTo= code.cloudfoundry.org/bytefmt v0.0.0-20230612151507-41ef4d1f67a4/go.mod h1:wYHCXH/gI19ujoFVuMkY48qPpPCoHLKBKXPkn67h/Yc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -161,8 +162,8 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/ajstarks/deck v0.0.0-20230917174535-ca5324ee72b2/go.mod h1:kSGTjaazWvDqs9hZd/UJu5WmdGOkYCDzQYOgQEa+v9Y= -github.com/ajstarks/deck/generate v0.0.0-20230917174535-ca5324ee72b2/go.mod h1:u04DhpZIpzaPnAUmhhjibCj450/2ITtLTEnvhaM49as= +github.com/ajstarks/deck v0.0.0-20231012031509-f833e437b68a/go.mod h1:CsIRRFEUjsvutRrtCL20i8EdMLo6JPw/F2XUmF+X694= +github.com/ajstarks/deck/generate v0.0.0-20231012031509-f833e437b68a/go.mod h1:u04DhpZIpzaPnAUmhhjibCj450/2ITtLTEnvhaM49as= github.com/ajstarks/fc v0.0.0-20230606144319-ef5d5cb73a3d/go.mod h1:Qp3TfzbBiIjHwDxIpu+g9nYfNw+xXF2Yqp4WmMlTtwM= github.com/ajstarks/openvg v0.0.0-20191008131700-c6885d824eb8/go.mod h1:jpZHIkd4sQEgrzshrUQrRfv5OUMMq0w/Q1yK6ZYhUlk= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw= @@ -176,46 +177,46 @@ github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/P github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.45.14 h1:/IPMEh9oelbK7506fdMHkbV9mq4a6f5aeiy9OT0PRKw= -github.com/aws/aws-sdk-go v1.45.14/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= -github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= -github.com/aws/aws-sdk-go-v2/config v1.18.41 h1:Go7z97YDsBJVNAaL7pDPKB6LeHEsAkHmFe+CeK30fUQ= -github.com/aws/aws-sdk-go-v2/config v1.18.41/go.mod h1:+yR45+A0LIMKT8bWOKo90Hy9rSrovEmEKoPKLmmVec8= -github.com/aws/aws-sdk-go-v2/credentials v1.13.39 h1:UnwBXDIHKDaejSXaRzKR57IdGCizk+z1DEhnsFpus7Q= -github.com/aws/aws-sdk-go-v2/credentials v1.13.39/go.mod h1:OJ9P239A90TnglJEF3qofKiNeEM6PCV/m+aNGV5WC24= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.85 h1:PH1OSkrdY8X+ENBz8ZhgWhR/975S05gcFPM4PxpAFrI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.85/go.mod h1:4X3OaVfB/ZjZsB9eHpFLnNT00sw+ZE81kQ2E9FysTaU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= +github.com/aws/aws-sdk-go v1.45.26 h1:PJ2NJNY5N/yeobLYe1Y+xLdavBi67ZI8gvph6ftwVCg= +github.com/aws/aws-sdk-go v1.45.26/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY= +github.com/aws/aws-sdk-go-v2/config v1.19.0 h1:AdzDvwH6dWuVARCl3RTLGRc4Ogy+N7yLFxVxXe1ClQ0= +github.com/aws/aws-sdk-go-v2/config v1.19.0/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91 h1:haAyxKHwoE+y/TJt+qHcPQf1dCViyyGbWcKjjYUllTE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.91/go.mod h1:ACQ6ta5YFlfSOz2c9A+EVYawLxFMZ0rI3Q0A0tGieKo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI= github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0/go.mod h1:EhC/83j8/hL/UB1WmExo3gkElaja/KlmZM/gl1rTfjM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.39.0 h1:VZ2WMkKLio5tVjYfThcy5+pb6YHGd6B6egq75FfM6hU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.39.0/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= -github.com/aws/aws-sdk-go-v2/service/sso v1.14.0 h1:AR/hlTsCyk1CwlyKnPFvIMvnONydRjDDRT9OGb0i+/g= -github.com/aws/aws-sdk-go-v2/service/sso v1.14.0/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 h1:7R8uRYyXzdD71KWVCL78lJZltah6VVznXBazvKjfH58= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:26SQUPcTNgV1Tapwdt4a1rOsYRsnBsJHLMPoxK2b0d8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 h1:9ulSU5ClouoPIYhDQdg9tpl83d5Yb91PXTKK+17q+ow= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 h1:Ll5/YVCOzRB+gxPqs2uD0R7/MyATC0w85626glSKmp4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2/go.mod h1:Zjfqt7KhQK+PO1bbOsFNzKgaq7TcxzmEoDWN8lM0qzQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg= -github.com/aws/aws-sdk-go-v2/service/sts v1.22.0 h1:s4bioTgjSFRwOoyEFzAVCmFmoowBgjTR8gkrF/sQ4wk= -github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= -github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= -github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= @@ -236,9 +237,9 @@ github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwys github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cncf/xds/go v0.0.0-20231016030527-8bd2eac9fb4a h1:SZL0tarhuhoN0kvo5pfO4i6vxYghwzXUo9w0WHIjI4k= +github.com/cncf/xds/go v0.0.0-20231016030527-8bd2eac9fb4a/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -322,8 +323,8 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= -github.com/gocraft/dbr/v2 v2.7.5 h1:TlXAEjDHazPKVsvUW9ZXug96B/vTqm+sSOCPb56rtTI= -github.com/gocraft/dbr/v2 v2.7.5/go.mod h1:8IH98S8M8J0JSEiYk0MPH26ZDUKemiQ/GvmXL5jo+Uw= +github.com/gocraft/dbr/v2 v2.7.6 h1:ASHKFgCbTLODbb9f756Cl8VAlnvQLKqIzx9E1Cfb7eo= +github.com/gocraft/dbr/v2 v2.7.6/go.mod h1:8IH98S8M8J0JSEiYk0MPH26ZDUKemiQ/GvmXL5jo+Uw= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -347,11 +348,14 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= -github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gnostic v0.7.0 h1:d7EpuFp8vVdML+y0JJJYiKeOLjKTdH/GvVkLOBWqJpw= +github.com/google/gnostic v0.7.0/go.mod h1:IAcUyMl6vtC95f60EZ8oXyqTsOersP6HbwjeG7EyDPM= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-pkcs11 v0.2.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= +github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE= github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= @@ -360,8 +364,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8 h1:gpptm606MZYGaMHMsB4Srmb6EbW/IVHnt04rcMXnkBQ= -github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= @@ -377,8 +381,9 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/ github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -425,8 +430,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.6.3/go.mod h1:nXw/i/MfnvRHqXa7XXmQMUB0oNFGuBrNI8d8NLy0LPw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.17.1-0.20230918170125-6125c11bae77 h1:02kW20OlAeZhbVue+Xriy4NWy9tskroqvX0PW1v/ACA= -github.com/klauspost/compress v1.17.1-0.20230918170125-6125c11bae77/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84= @@ -482,14 +487,14 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= -github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= +github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -508,14 +513,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/quasilyte/go-ruleguard v0.4.0 h1:DyM6r+TKL+xbKB4Nm7Afd1IQh9kEUKQs2pboWGKtvQo= github.com/quasilyte/go-ruleguard v0.4.0/go.mod h1:Eu76Z/R8IXtViWUIHkE3p8gdH3/PKk1eh3YGfaEof10= github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE= @@ -543,7 +548,7 @@ github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1 github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92/go.mod h1:7/OT02F6S6I7v6WXb+IjhMuZEYfH/RJ5RwEWnEo5BMg= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -598,8 +603,8 @@ go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZX go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.starlark.net v0.0.0-20230912135651-745481cf39ed h1:kNt8RXSIU6IRBO9MP3m+6q3WpyBHQQXqSktcyVKDPOQ= -go.starlark.net v0.0.0-20230912135651-745481cf39ed/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.starlark.net v0.0.0-20231016134836-22325403fcb3 h1:CKbpFNZNfaNyEWd6C+F1vLZ0WJjukoU45zDErBmRKPs= +go.starlark.net v0.0.0-20231016134836-22325403fcb3/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -610,39 +615,39 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= gocloud.dev v0.34.0 h1:LzlQY+4l2cMtuNfwT2ht4+fiXwWf/NmPTnXUlLmGif4= gocloud.dev v0.34.0/go.mod h1:psKOachbnvY3DAOPbsFVmLIErwsbWPUG2H5i65D38vE= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/exp/shiny v0.0.0-20220827204233-334a2380cb91/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8= golang.org/x/exp/shiny v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= golang.org/x/exp/shiny v0.0.0-20230817173708-d852ddb80c63/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= -golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 h1:j3D9DvWRpUfIyFfDPws7LoIZ2MAI1OJHdQXtTnYtN+k= -golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ= -golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk= +golang.org/x/exp/typeparams v0.0.0-20231006140011-7918f672742d h1:NRn/Afz91uVUyEsxMp4lGGxpr5y1qz+Iko60dbkfvLQ= +golang.org/x/exp/typeparams v0.0.0-20231006140011-7918f672742d/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= +golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20230906132913-2077a3224571/go.mod h1:wEyOn6VvNW7tcf+bW/wBz1sehi2s2BZ4TimyR7qZen4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/mobile v0.0.0-20231006135142-2b44d11868fe/go.mod h1:BrnXpEObnFxpaT75Jo9hsCazwOWcp7nVIa8NNuH5cuA= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0= @@ -651,19 +656,19 @@ gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENE gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= gonum.org/v1/plot v0.14.0 h1:+LBDVFYwFe4LHhdP8coW6296MBEY4nQ+Y4vuUpJopcE= gonum.org/v1/plot v0.14.0/go.mod h1:MLdR9424SJed+5VqC6MsouEpig9pZX2VZ57H9ko2bXU= -google.golang.org/api v0.142.0 h1:mf+7EJ94fi5ZcnpPy+m0Yv2dkz8bKm+UL0snTCuwXlY= -google.golang.org/api v0.142.0/go.mod h1:zJAN5o6HRqR7O+9qJUFOWrZkYE66RH+efPBdTLA4xBA= +google.golang.org/api v0.147.0 h1:Can3FaQo9LlVqxJCodNmeZW/ib3/qKAY3rFeXiHo5gc= +google.golang.org/api v0.147.0/go.mod h1:pQ/9j83DcmPd/5C9e2nFOdjjNkDZ1G+zkbK2uvdkJMs= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 h1:vlzZttNJGVqTsRFU9AmdnrcO1Znh8Ew9kCD//yjigk0= -google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:CCviP9RmpZ1mxVr8MUjCnSiY09IbAXZxhLE6EhHIdPU= -google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 h1:U7+wNaVuSTaUqNvK2+osJ9ejEZxbjHHk8F2b6Hpx0AE= -google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230913181813-007df8e322eb/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= -google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= -google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:itlFWGBbEyD32PUeJsTG8h8Wz7iJXfVK4gt1EJ+pAG0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/hack/go.mod.default b/hack/go.mod.default index fa261d6dcb..b78b7c3c84 100755 --- a/hack/go.mod.default +++ b/hack/go.mod.default @@ -3,275 +3,275 @@ module github.com/vdaas/vald go 1.21 replace ( - cloud.google.com/go => cloud.google.com/go latest - cloud.google.com/go/bigquery => cloud.google.com/go/bigquery latest - cloud.google.com/go/compute => cloud.google.com/go/compute latest - cloud.google.com/go/datastore => cloud.google.com/go/datastore latest - cloud.google.com/go/firestore => cloud.google.com/go/firestore latest - cloud.google.com/go/iam => cloud.google.com/go/iam latest - cloud.google.com/go/kms => cloud.google.com/go/kms latest - cloud.google.com/go/monitoring => cloud.google.com/go/monitoring latest - cloud.google.com/go/pubsub => cloud.google.com/go/pubsub latest - cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager latest - cloud.google.com/go/storage => cloud.google.com/go/storage latest - cloud.google.com/go/trace => cloud.google.com/go/trace latest - code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt latest - contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws latest - contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus latest - contrib.go.opencensus.io/integrations/ocsql => contrib.go.opencensus.io/integrations/ocsql latest - git.sr.ht/~sbinet/gg => git.sr.ht/~sbinet/gg latest - github.com/Azure/azure-amqp-common-go/v3 => github.com/Azure/azure-amqp-common-go/v3 latest - github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go latest - github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore latest - github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity latest - github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal latest - github.com/Azure/go-amqp => github.com/Azure/go-amqp latest + cloud.google.com/go => cloud.google.com/go upgrade + cloud.google.com/go/bigquery => cloud.google.com/go/bigquery upgrade + cloud.google.com/go/compute => cloud.google.com/go/compute upgrade + cloud.google.com/go/datastore => cloud.google.com/go/datastore upgrade + cloud.google.com/go/firestore => cloud.google.com/go/firestore upgrade + cloud.google.com/go/iam => cloud.google.com/go/iam upgrade + cloud.google.com/go/kms => cloud.google.com/go/kms upgrade + cloud.google.com/go/monitoring => cloud.google.com/go/monitoring upgrade + cloud.google.com/go/pubsub => cloud.google.com/go/pubsub upgrade + cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager upgrade + cloud.google.com/go/storage => cloud.google.com/go/storage upgrade + cloud.google.com/go/trace => cloud.google.com/go/trace upgrade + code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt upgrade + contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws upgrade + contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus upgrade + contrib.go.opencensus.io/integrations/ocsql => contrib.go.opencensus.io/integrations/ocsql upgrade + git.sr.ht/~sbinet/gg => git.sr.ht/~sbinet/gg upgrade + github.com/Azure/azure-amqp-common-go/v3 => github.com/Azure/azure-amqp-common-go/v3 upgrade + github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go upgrade + github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore upgrade + github.com/Azure/azure-sdk-for-go/sdk/azidentity => github.com/Azure/azure-sdk-for-go/sdk/azidentity upgrade + github.com/Azure/azure-sdk-for-go/sdk/internal => github.com/Azure/azure-sdk-for-go/sdk/internal upgrade + github.com/Azure/go-amqp => github.com/Azure/go-amqp upgrade github.com/Azure/go-autorest => github.com/Azure/go-autorest main github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/autorest main - github.com/Azure/go-autorest/autorest/adal => github.com/Azure/go-autorest/autorest/adal latest + github.com/Azure/go-autorest/autorest/adal => github.com/Azure/go-autorest/autorest/adal upgrade github.com/Azure/go-autorest/autorest/date => github.com/Azure/go-autorest/autorest/date main github.com/Azure/go-autorest/autorest/mocks => github.com/Azure/go-autorest/autorest/mocks main github.com/Azure/go-autorest/autorest/to => github.com/Azure/go-autorest/autorest/to main github.com/Azure/go-autorest/logger => github.com/Azure/go-autorest/logger main github.com/Azure/go-autorest/tracing => github.com/Azure/go-autorest/tracing main - github.com/BurntSushi/toml => github.com/BurntSushi/toml latest - github.com/DATA-DOG/go-sqlmock => github.com/DATA-DOG/go-sqlmock latest - github.com/GoogleCloudPlatform/cloudsql-proxy => github.com/GoogleCloudPlatform/cloudsql-proxy latest - github.com/Masterminds/semver/v3 => github.com/Masterminds/semver/v3 latest - github.com/ajstarks/deck => github.com/ajstarks/deck latest - github.com/ajstarks/deck/generate => github.com/ajstarks/deck/generate latest - github.com/ajstarks/svgo => github.com/ajstarks/svgo latest - github.com/antihax/optional => github.com/antihax/optional latest - github.com/armon/go-socks5 => github.com/armon/go-socks5 latest - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go latest - github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 latest - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream latest - github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config latest - github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials latest - github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds latest - github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager latest - github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources latest - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 latest - github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini latest - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding latest - github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum latest - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url latest - github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared latest - github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms latest - github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 latest - github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager latest - github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns latest - github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs latest - github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm latest - github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso latest - github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts latest - github.com/aws/smithy-go => github.com/aws/smithy-go latest - github.com/benbjohnson/clock => github.com/benbjohnson/clock latest - github.com/beorn7/perks => github.com/beorn7/perks latest - github.com/bmizerany/assert => github.com/bmizerany/assert latest - github.com/boombuler/barcode => github.com/boombuler/barcode latest - github.com/buger/jsonparser => github.com/buger/jsonparser latest - github.com/cenkalti/backoff/v4 => github.com/cenkalti/backoff/v4 latest - github.com/census-instrumentation/opencensus-proto => github.com/census-instrumentation/opencensus-proto latest - github.com/cespare/xxhash/v2 => github.com/cespare/xxhash/v2 latest - github.com/chzyer/logex => github.com/chzyer/logex latest - github.com/chzyer/readline => github.com/chzyer/readline latest - github.com/chzyer/test => github.com/chzyer/test latest - github.com/cncf/udpa/go => github.com/cncf/udpa/go latest - github.com/cncf/xds/go => github.com/cncf/xds/go latest - github.com/cockroachdb/apd => github.com/cockroachdb/apd latest - github.com/coreos/go-systemd/v22 => github.com/coreos/go-systemd/v22 latest - github.com/cpuguy83/go-md2man/v2 => github.com/cpuguy83/go-md2man/v2 latest - github.com/creack/pty => github.com/creack/pty latest - github.com/davecgh/go-spew => github.com/davecgh/go-spew latest - github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb latest - github.com/devigned/tab => github.com/devigned/tab latest - github.com/dgryski/go-rendezvous => github.com/dgryski/go-rendezvous latest - github.com/dnaeon/go-vcr => github.com/dnaeon/go-vcr latest - github.com/docopt/docopt-go => github.com/docopt/docopt-go latest - github.com/dustin/go-humanize => github.com/dustin/go-humanize latest - github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 latest - github.com/envoyproxy/go-control-plane => github.com/envoyproxy/go-control-plane latest - github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate latest - github.com/evanphx/json-patch => github.com/evanphx/json-patch latest - github.com/fogleman/gg => github.com/fogleman/gg latest - github.com/fortytw2/leaktest => github.com/fortytw2/leaktest latest - github.com/frankban/quicktest => github.com/frankban/quicktest latest - github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify latest - github.com/gin-contrib/sse => github.com/gin-contrib/sse latest - github.com/gin-gonic/gin => github.com/gin-gonic/gin latest - github.com/go-errors/errors => github.com/go-errors/errors latest - github.com/go-fonts/dejavu => github.com/go-fonts/dejavu latest - github.com/go-fonts/latin-modern => github.com/go-fonts/latin-modern latest - github.com/go-fonts/liberation => github.com/go-fonts/liberation latest - github.com/go-fonts/stix => github.com/go-fonts/stix latest - github.com/go-gl/gl => github.com/go-gl/gl latest - github.com/go-gl/glfw/v3.3/glfw => github.com/go-gl/glfw/v3.3/glfw latest - github.com/go-kit/log => github.com/go-kit/log latest - github.com/go-latex/latex => github.com/go-latex/latex latest - github.com/go-logfmt/logfmt => github.com/go-logfmt/logfmt latest - github.com/go-logr/logr => github.com/go-logr/logr latest - github.com/go-logr/stdr => github.com/go-logr/stdr latest - github.com/go-logr/zapr => github.com/go-logr/zapr latest - github.com/go-openapi/jsonpointer => github.com/go-openapi/jsonpointer latest - github.com/go-openapi/jsonreference => github.com/go-openapi/jsonreference latest - github.com/go-openapi/swag => github.com/go-openapi/swag latest - github.com/go-pdf/fpdf => github.com/go-pdf/fpdf latest - github.com/go-playground/assert/v2 => github.com/go-playground/assert/v2 latest - github.com/go-playground/locales => github.com/go-playground/locales latest - github.com/go-playground/universal-translator => github.com/go-playground/universal-translator latest - github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 latest - github.com/go-redis/redis/v8 => github.com/go-redis/redis/v8 latest - github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql latest - github.com/go-task/slim-sprig => github.com/go-task/slim-sprig latest - github.com/go-toolsmith/astcopy => github.com/go-toolsmith/astcopy latest - github.com/go-toolsmith/astequal => github.com/go-toolsmith/astequal latest - github.com/go-toolsmith/strparse => github.com/go-toolsmith/strparse latest - github.com/gobwas/httphead => github.com/gobwas/httphead latest - github.com/gobwas/pool => github.com/gobwas/pool latest - github.com/gobwas/ws => github.com/gobwas/ws latest - github.com/goccy/go-json => github.com/goccy/go-json latest - github.com/gocql/gocql => github.com/gocql/gocql latest - github.com/gocraft/dbr/v2 => github.com/gocraft/dbr/v2 latest - github.com/godbus/dbus/v5 => github.com/godbus/dbus/v5 latest - github.com/gofrs/uuid => github.com/gofrs/uuid latest - github.com/gogo/protobuf => github.com/gogo/protobuf latest - github.com/golang-jwt/jwt/v4 => github.com/golang-jwt/jwt/v4 latest - github.com/golang-sql/civil => github.com/golang-sql/civil latest - github.com/golang-sql/sqlexp => github.com/golang-sql/sqlexp latest - github.com/golang/freetype => github.com/golang/freetype latest - github.com/golang/glog => github.com/golang/glog latest - github.com/golang/groupcache => github.com/golang/groupcache latest - github.com/golang/mock => github.com/golang/mock latest - github.com/golang/protobuf => github.com/golang/protobuf latest - github.com/golang/snappy => github.com/golang/snappy latest + github.com/BurntSushi/toml => github.com/BurntSushi/toml upgrade + github.com/DATA-DOG/go-sqlmock => github.com/DATA-DOG/go-sqlmock upgrade + github.com/GoogleCloudPlatform/cloudsql-proxy => github.com/GoogleCloudPlatform/cloudsql-proxy upgrade + github.com/Masterminds/semver/v3 => github.com/Masterminds/semver/v3 upgrade + github.com/ajstarks/deck => github.com/ajstarks/deck upgrade + github.com/ajstarks/deck/generate => github.com/ajstarks/deck/generate upgrade + github.com/ajstarks/svgo => github.com/ajstarks/svgo upgrade + github.com/antihax/optional => github.com/antihax/optional upgrade + github.com/armon/go-socks5 => github.com/armon/go-socks5 upgrade + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go upgrade + github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 upgrade + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream upgrade + github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config upgrade + github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials upgrade + github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds upgrade + github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager upgrade + github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources upgrade + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 upgrade + github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini upgrade + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding => github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding upgrade + github.com/aws/aws-sdk-go-v2/service/internal/checksum => github.com/aws/aws-sdk-go-v2/service/internal/checksum upgrade + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url => github.com/aws/aws-sdk-go-v2/service/internal/presigned-url upgrade + github.com/aws/aws-sdk-go-v2/service/internal/s3shared => github.com/aws/aws-sdk-go-v2/service/internal/s3shared upgrade + github.com/aws/aws-sdk-go-v2/service/kms => github.com/aws/aws-sdk-go-v2/service/kms upgrade + github.com/aws/aws-sdk-go-v2/service/s3 => github.com/aws/aws-sdk-go-v2/service/s3 upgrade + github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager upgrade + github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns upgrade + github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs upgrade + github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm upgrade + github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso upgrade + github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts upgrade + github.com/aws/smithy-go => github.com/aws/smithy-go upgrade + github.com/benbjohnson/clock => github.com/benbjohnson/clock upgrade + github.com/beorn7/perks => github.com/beorn7/perks upgrade + github.com/bmizerany/assert => github.com/bmizerany/assert upgrade + github.com/boombuler/barcode => github.com/boombuler/barcode upgrade + github.com/buger/jsonparser => github.com/buger/jsonparser upgrade + github.com/cenkalti/backoff/v4 => github.com/cenkalti/backoff/v4 upgrade + github.com/census-instrumentation/opencensus-proto => github.com/census-instrumentation/opencensus-proto upgrade + github.com/cespare/xxhash/v2 => github.com/cespare/xxhash/v2 upgrade + github.com/chzyer/logex => github.com/chzyer/logex upgrade + github.com/chzyer/readline => github.com/chzyer/readline upgrade + github.com/chzyer/test => github.com/chzyer/test upgrade + github.com/cncf/udpa/go => github.com/cncf/udpa/go upgrade + github.com/cncf/xds/go => github.com/cncf/xds/go upgrade + github.com/cockroachdb/apd => github.com/cockroachdb/apd upgrade + github.com/coreos/go-systemd/v22 => github.com/coreos/go-systemd/v22 upgrade + github.com/cpuguy83/go-md2man/v2 => github.com/cpuguy83/go-md2man/v2 upgrade + github.com/creack/pty => github.com/creack/pty upgrade + github.com/davecgh/go-spew => github.com/davecgh/go-spew upgrade + github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb upgrade + github.com/devigned/tab => github.com/devigned/tab upgrade + github.com/dgryski/go-rendezvous => github.com/dgryski/go-rendezvous upgrade + github.com/dnaeon/go-vcr => github.com/dnaeon/go-vcr upgrade + github.com/docopt/docopt-go => github.com/docopt/docopt-go upgrade + github.com/dustin/go-humanize => github.com/dustin/go-humanize upgrade + github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 upgrade + github.com/envoyproxy/go-control-plane => github.com/envoyproxy/go-control-plane upgrade + github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate upgrade + github.com/evanphx/json-patch => github.com/evanphx/json-patch upgrade + github.com/fogleman/gg => github.com/fogleman/gg upgrade + github.com/fortytw2/leaktest => github.com/fortytw2/leaktest upgrade + github.com/frankban/quicktest => github.com/frankban/quicktest upgrade + github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify upgrade + github.com/gin-contrib/sse => github.com/gin-contrib/sse upgrade + github.com/gin-gonic/gin => github.com/gin-gonic/gin upgrade + github.com/go-errors/errors => github.com/go-errors/errors upgrade + github.com/go-fonts/dejavu => github.com/go-fonts/dejavu upgrade + github.com/go-fonts/latin-modern => github.com/go-fonts/latin-modern upgrade + github.com/go-fonts/liberation => github.com/go-fonts/liberation upgrade + github.com/go-fonts/stix => github.com/go-fonts/stix upgrade + github.com/go-gl/gl => github.com/go-gl/gl upgrade + github.com/go-gl/glfw/v3.3/glfw => github.com/go-gl/glfw/v3.3/glfw upgrade + github.com/go-kit/log => github.com/go-kit/log upgrade + github.com/go-latex/latex => github.com/go-latex/latex upgrade + github.com/go-logfmt/logfmt => github.com/go-logfmt/logfmt upgrade + github.com/go-logr/logr => github.com/go-logr/logr upgrade + github.com/go-logr/stdr => github.com/go-logr/stdr upgrade + github.com/go-logr/zapr => github.com/go-logr/zapr upgrade + github.com/go-openapi/jsonpointer => github.com/go-openapi/jsonpointer upgrade + github.com/go-openapi/jsonreference => github.com/go-openapi/jsonreference upgrade + github.com/go-openapi/swag => github.com/go-openapi/swag upgrade + github.com/go-pdf/fpdf => github.com/go-pdf/fpdf upgrade + github.com/go-playground/assert/v2 => github.com/go-playground/assert/v2 upgrade + github.com/go-playground/locales => github.com/go-playground/locales upgrade + github.com/go-playground/universal-translator => github.com/go-playground/universal-translator upgrade + github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 upgrade + github.com/go-redis/redis/v8 => github.com/go-redis/redis/v8 upgrade + github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql upgrade + github.com/go-task/slim-sprig => github.com/go-task/slim-sprig upgrade + github.com/go-toolsmith/astcopy => github.com/go-toolsmith/astcopy upgrade + github.com/go-toolsmith/astequal => github.com/go-toolsmith/astequal upgrade + github.com/go-toolsmith/strparse => github.com/go-toolsmith/strparse upgrade + github.com/gobwas/httphead => github.com/gobwas/httphead upgrade + github.com/gobwas/pool => github.com/gobwas/pool upgrade + github.com/gobwas/ws => github.com/gobwas/ws upgrade + github.com/goccy/go-json => github.com/goccy/go-json upgrade + github.com/gocql/gocql => github.com/gocql/gocql upgrade + github.com/gocraft/dbr/v2 => github.com/gocraft/dbr/v2 upgrade + github.com/godbus/dbus/v5 => github.com/godbus/dbus/v5 upgrade + github.com/gofrs/uuid => github.com/gofrs/uuid upgrade + github.com/gogo/protobuf => github.com/gogo/protobuf upgrade + github.com/golang-jwt/jwt/v4 => github.com/golang-jwt/jwt/v4 upgrade + github.com/golang-sql/civil => github.com/golang-sql/civil upgrade + github.com/golang-sql/sqlexp => github.com/golang-sql/sqlexp upgrade + github.com/golang/freetype => github.com/golang/freetype upgrade + github.com/golang/glog => github.com/golang/glog upgrade + github.com/golang/groupcache => github.com/golang/groupcache upgrade + github.com/golang/mock => github.com/golang/mock upgrade + github.com/golang/protobuf => github.com/golang/protobuf upgrade + github.com/golang/snappy => github.com/golang/snappy upgrade github.com/google/btree => github.com/google/btree master - github.com/google/gnostic => github.com/google/gnostic latest - github.com/google/go-cmp => github.com/google/go-cmp latest - github.com/google/go-replayers/grpcreplay => github.com/google/go-replayers/grpcreplay latest - github.com/google/go-replayers/httpreplay => github.com/google/go-replayers/httpreplay latest - github.com/google/gofuzz => github.com/google/gofuzz latest - github.com/google/martian => github.com/google/martian latest - github.com/google/martian/v3 => github.com/google/martian/v3 latest - github.com/google/pprof => github.com/google/pprof latest - github.com/google/shlex => github.com/google/shlex latest - github.com/google/subcommands => github.com/google/subcommands latest - github.com/google/uuid => github.com/google/uuid latest - github.com/google/wire => github.com/google/wire latest - github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 latest - github.com/gorilla/mux => github.com/gorilla/mux latest - github.com/gorilla/websocket => github.com/gorilla/websocket latest - github.com/gregjones/httpcache => github.com/gregjones/httpcache latest - github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/grpc-gateway/v2 latest - github.com/hailocab/go-hostpool => github.com/kpango/go-hostpool latest - github.com/hanwen/go-fuse/v2 => github.com/hanwen/go-fuse/v2 latest - github.com/hashicorp/go-uuid => github.com/hashicorp/go-uuid latest - github.com/hashicorp/go-version => github.com/hashicorp/go-version latest - github.com/iancoleman/strcase => github.com/iancoleman/strcase latest - github.com/ianlancetaylor/demangle => github.com/ianlancetaylor/demangle latest - github.com/inconshreveable/mousetrap => github.com/inconshreveable/mousetrap latest - github.com/jackc/chunkreader/v2 => github.com/jackc/chunkreader/v2 latest - github.com/jackc/pgconn => github.com/jackc/pgconn latest - github.com/jackc/pgio => github.com/jackc/pgio latest - github.com/jackc/pgmock => github.com/jackc/pgmock latest - github.com/jackc/pgpassfile => github.com/jackc/pgpassfile latest - github.com/jackc/pgproto3/v2 => github.com/jackc/pgproto3/v2 latest - github.com/jackc/pgservicefile => github.com/jackc/pgservicefile latest - github.com/jackc/pgtype => github.com/jackc/pgtype latest - github.com/jackc/pgx/v4 => github.com/jackc/pgx/v4 latest - github.com/jackc/puddle => github.com/jackc/puddle latest - github.com/jessevdk/go-flags => github.com/jessevdk/go-flags latest - github.com/jmespath/go-jmespath => github.com/jmespath/go-jmespath latest - github.com/jmespath/go-jmespath/internal/testify => github.com/jmespath/go-jmespath/internal/testify latest - github.com/jmoiron/sqlx => github.com/jmoiron/sqlx latest - github.com/joho/godotenv => github.com/joho/godotenv latest - github.com/josharian/intern => github.com/josharian/intern latest - github.com/json-iterator/go => github.com/json-iterator/go latest - github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report latest - github.com/kisielk/errcheck => github.com/kisielk/errcheck latest - github.com/kisielk/gotool => github.com/kisielk/gotool latest + github.com/google/gnostic => github.com/google/gnostic upgrade + github.com/google/go-cmp => github.com/google/go-cmp upgrade + github.com/google/go-replayers/grpcreplay => github.com/google/go-replayers/grpcreplay upgrade + github.com/google/go-replayers/httpreplay => github.com/google/go-replayers/httpreplay upgrade + github.com/google/gofuzz => github.com/google/gofuzz upgrade + github.com/google/martian => github.com/google/martian upgrade + github.com/google/martian/v3 => github.com/google/martian/v3 upgrade + github.com/google/pprof => github.com/google/pprof upgrade + github.com/google/shlex => github.com/google/shlex upgrade + github.com/google/subcommands => github.com/google/subcommands upgrade + github.com/google/uuid => github.com/google/uuid upgrade + github.com/google/wire => github.com/google/wire upgrade + github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 upgrade + github.com/gorilla/mux => github.com/gorilla/mux upgrade + github.com/gorilla/websocket => github.com/gorilla/websocket upgrade + github.com/gregjones/httpcache => github.com/gregjones/httpcache upgrade + github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/grpc-gateway/v2 upgrade + github.com/hailocab/go-hostpool => github.com/kpango/go-hostpool upgrade + github.com/hanwen/go-fuse/v2 => github.com/hanwen/go-fuse/v2 upgrade + github.com/hashicorp/go-uuid => github.com/hashicorp/go-uuid upgrade + github.com/hashicorp/go-version => github.com/hashicorp/go-version upgrade + github.com/iancoleman/strcase => github.com/iancoleman/strcase upgrade + github.com/ianlancetaylor/demangle => github.com/ianlancetaylor/demangle upgrade + github.com/inconshreveable/mousetrap => github.com/inconshreveable/mousetrap upgrade + github.com/jackc/chunkreader/v2 => github.com/jackc/chunkreader/v2 upgrade + github.com/jackc/pgconn => github.com/jackc/pgconn upgrade + github.com/jackc/pgio => github.com/jackc/pgio upgrade + github.com/jackc/pgmock => github.com/jackc/pgmock upgrade + github.com/jackc/pgpassfile => github.com/jackc/pgpassfile upgrade + github.com/jackc/pgproto3/v2 => github.com/jackc/pgproto3/v2 upgrade + github.com/jackc/pgservicefile => github.com/jackc/pgservicefile upgrade + github.com/jackc/pgtype => github.com/jackc/pgtype upgrade + github.com/jackc/pgx/v4 => github.com/jackc/pgx/v4 upgrade + github.com/jackc/puddle => github.com/jackc/puddle upgrade + github.com/jessevdk/go-flags => github.com/jessevdk/go-flags upgrade + github.com/jmespath/go-jmespath => github.com/jmespath/go-jmespath upgrade + github.com/jmespath/go-jmespath/internal/testify => github.com/jmespath/go-jmespath/internal/testify upgrade + github.com/jmoiron/sqlx => github.com/jmoiron/sqlx upgrade + github.com/joho/godotenv => github.com/joho/godotenv upgrade + github.com/josharian/intern => github.com/josharian/intern upgrade + github.com/json-iterator/go => github.com/json-iterator/go upgrade + github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report upgrade + github.com/kisielk/errcheck => github.com/kisielk/errcheck upgrade + github.com/kisielk/gotool => github.com/kisielk/gotool upgrade github.com/klauspost/compress => github.com/klauspost/compress master - github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 latest - github.com/kpango/fastime => github.com/kpango/fastime latest - github.com/kpango/fuid => github.com/kpango/fuid latest - github.com/kpango/gache/v2 => github.com/kpango/gache/v2 latest - github.com/kpango/glg => github.com/kpango/glg latest - github.com/kr/fs => github.com/kr/fs latest - github.com/kr/pretty => github.com/kr/pretty latest - github.com/kr/text => github.com/kr/text latest - github.com/kylelemons/godebug => github.com/kylelemons/godebug latest - github.com/leanovate/gopter => github.com/leanovate/gopter latest - github.com/leodido/go-urn => github.com/leodido/go-urn latest - github.com/lib/pq => github.com/lib/pq latest - github.com/liggitt/tabwriter => github.com/liggitt/tabwriter latest - github.com/lucasb-eyer/go-colorful => github.com/lucasb-eyer/go-colorful latest - github.com/mailru/easyjson => github.com/mailru/easyjson latest - github.com/mattn/go-colorable => github.com/mattn/go-colorable latest - github.com/mattn/go-isatty => github.com/mattn/go-isatty latest - github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 latest - github.com/matttproud/golang_protobuf_extensions => github.com/matttproud/golang_protobuf_extensions latest - github.com/mitchellh/colorstring => github.com/mitchellh/colorstring latest - github.com/moby/spdystream => github.com/moby/spdystream latest - github.com/moby/sys/mountinfo => github.com/moby/sys/mountinfo latest - github.com/modern-go/concurrent => github.com/modern-go/concurrent latest - github.com/modern-go/reflect2 => github.com/modern-go/reflect2 latest - github.com/modocache/gover => github.com/modocache/gover latest - github.com/monochromegane/go-gitignore => github.com/monochromegane/go-gitignore latest - github.com/montanaflynn/stats => github.com/montanaflynn/stats latest - github.com/munnerz/goautoneg => github.com/munnerz/goautoneg latest - github.com/niemeyer/pretty => github.com/niemeyer/pretty latest - github.com/nxadm/tail => github.com/nxadm/tail latest - github.com/onsi/ginkgo => github.com/onsi/ginkgo latest - github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 latest - github.com/onsi/gomega => github.com/onsi/gomega latest - github.com/peterbourgon/diskv => github.com/peterbourgon/diskv latest - github.com/phpdave11/gofpdf => github.com/phpdave11/gofpdf latest - github.com/phpdave11/gofpdi => github.com/phpdave11/gofpdi latest - github.com/pierrec/cmdflag => github.com/pierrec/cmdflag latest - github.com/pierrec/lz4/v3 => github.com/pierrec/lz4/v3 latest - github.com/pkg/browser => github.com/pkg/browser latest - github.com/pkg/errors => github.com/pkg/errors latest - github.com/pkg/sftp => github.com/pkg/sftp latest - github.com/pmezard/go-difflib => github.com/pmezard/go-difflib latest - github.com/prashantv/gostub => github.com/prashantv/gostub latest - github.com/prometheus/client_golang => github.com/prometheus/client_golang latest - github.com/prometheus/client_model => github.com/prometheus/client_model latest - github.com/prometheus/common => github.com/prometheus/common latest - github.com/prometheus/procfs => github.com/prometheus/procfs latest - github.com/prometheus/prometheus => github.com/prometheus/prometheus latest - github.com/quasilyte/go-ruleguard => github.com/quasilyte/go-ruleguard latest - github.com/quasilyte/go-ruleguard/dsl => github.com/quasilyte/go-ruleguard/dsl latest - github.com/quasilyte/gogrep => github.com/quasilyte/gogrep latest - github.com/quasilyte/stdinfo => github.com/quasilyte/stdinfo latest - github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid latest - github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal latest - github.com/rs/xid => github.com/rs/xid latest - github.com/rs/zerolog => github.com/rs/zerolog latest - github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 latest - github.com/ruudk/golang-pdf417 => github.com/ruudk/golang-pdf417 latest - github.com/schollz/progressbar/v2 => github.com/schollz/progressbar/v2 latest - github.com/scylladb/go-reflectx => github.com/scylladb/go-reflectx latest - github.com/scylladb/gocqlx => github.com/scylladb/gocqlx latest - github.com/sergi/go-diff => github.com/sergi/go-diff latest - github.com/shopspring/decimal => github.com/shopspring/decimal latest - github.com/shurcooL/httpfs => github.com/shurcooL/httpfs latest - github.com/shurcooL/vfsgen => github.com/shurcooL/vfsgen latest - github.com/sirupsen/logrus => github.com/sirupsen/logrus latest - github.com/spf13/afero => github.com/spf13/afero latest - github.com/spf13/cobra => github.com/spf13/cobra latest - github.com/spf13/pflag => github.com/spf13/pflag latest - github.com/stoewer/go-strcase => github.com/stoewer/go-strcase latest - github.com/stretchr/objx => github.com/stretchr/objx latest - github.com/stretchr/testify => github.com/stretchr/testify latest - github.com/ugorji/go/codec => github.com/ugorji/go/codec latest - github.com/xeipuuv/gojsonpointer => github.com/xeipuuv/gojsonpointer latest - github.com/xeipuuv/gojsonreference => github.com/xeipuuv/gojsonreference latest - github.com/xeipuuv/gojsonschema => github.com/xeipuuv/gojsonschema latest - github.com/xlab/treeprint => github.com/xlab/treeprint latest - github.com/zeebo/assert => github.com/zeebo/assert latest - github.com/zeebo/xxh3 => github.com/zeebo/xxh3 latest - go.opencensus.io => go.opencensus.io latest - go.etcd.io/bbolt => go.etcd.io/bbolt latest + github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 upgrade + github.com/kpango/fastime => github.com/kpango/fastime upgrade + github.com/kpango/fuid => github.com/kpango/fuid upgrade + github.com/kpango/gache/v2 => github.com/kpango/gache/v2 upgrade + github.com/kpango/glg => github.com/kpango/glg upgrade + github.com/kr/fs => github.com/kr/fs upgrade + github.com/kr/pretty => github.com/kr/pretty upgrade + github.com/kr/text => github.com/kr/text upgrade + github.com/kylelemons/godebug => github.com/kylelemons/godebug upgrade + github.com/leanovate/gopter => github.com/leanovate/gopter upgrade + github.com/leodido/go-urn => github.com/leodido/go-urn upgrade + github.com/lib/pq => github.com/lib/pq upgrade + github.com/liggitt/tabwriter => github.com/liggitt/tabwriter upgrade + github.com/lucasb-eyer/go-colorful => github.com/lucasb-eyer/go-colorful upgrade + github.com/mailru/easyjson => github.com/mailru/easyjson upgrade + github.com/mattn/go-colorable => github.com/mattn/go-colorable upgrade + github.com/mattn/go-isatty => github.com/mattn/go-isatty upgrade + github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 upgrade + github.com/matttproud/golang_protobuf_extensions => github.com/matttproud/golang_protobuf_extensions upgrade + github.com/mitchellh/colorstring => github.com/mitchellh/colorstring upgrade + github.com/moby/spdystream => github.com/moby/spdystream upgrade + github.com/moby/sys/mountinfo => github.com/moby/sys/mountinfo upgrade + github.com/modern-go/concurrent => github.com/modern-go/concurrent upgrade + github.com/modern-go/reflect2 => github.com/modern-go/reflect2 upgrade + github.com/modocache/gover => github.com/modocache/gover upgrade + github.com/monochromegane/go-gitignore => github.com/monochromegane/go-gitignore upgrade + github.com/montanaflynn/stats => github.com/montanaflynn/stats upgrade + github.com/munnerz/goautoneg => github.com/munnerz/goautoneg upgrade + github.com/niemeyer/pretty => github.com/niemeyer/pretty upgrade + github.com/nxadm/tail => github.com/nxadm/tail upgrade + github.com/onsi/ginkgo => github.com/onsi/ginkgo upgrade + github.com/onsi/ginkgo/v2 => github.com/onsi/ginkgo/v2 upgrade + github.com/onsi/gomega => github.com/onsi/gomega upgrade + github.com/peterbourgon/diskv => github.com/peterbourgon/diskv upgrade + github.com/phpdave11/gofpdf => github.com/phpdave11/gofpdf upgrade + github.com/phpdave11/gofpdi => github.com/phpdave11/gofpdi upgrade + github.com/pierrec/cmdflag => github.com/pierrec/cmdflag upgrade + github.com/pierrec/lz4/v3 => github.com/pierrec/lz4/v3 upgrade + github.com/pkg/browser => github.com/pkg/browser upgrade + github.com/pkg/errors => github.com/pkg/errors upgrade + github.com/pkg/sftp => github.com/pkg/sftp upgrade + github.com/pmezard/go-difflib => github.com/pmezard/go-difflib upgrade + github.com/prashantv/gostub => github.com/prashantv/gostub upgrade + github.com/prometheus/client_golang => github.com/prometheus/client_golang upgrade + github.com/prometheus/client_model => github.com/prometheus/client_model upgrade + github.com/prometheus/common => github.com/prometheus/common upgrade + github.com/prometheus/procfs => github.com/prometheus/procfs upgrade + github.com/prometheus/prometheus => github.com/prometheus/prometheus upgrade + github.com/quasilyte/go-ruleguard => github.com/quasilyte/go-ruleguard upgrade + github.com/quasilyte/go-ruleguard/dsl => github.com/quasilyte/go-ruleguard/dsl upgrade + github.com/quasilyte/gogrep => github.com/quasilyte/gogrep upgrade + github.com/quasilyte/stdinfo => github.com/quasilyte/stdinfo upgrade + github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid upgrade + github.com/rogpeppe/go-internal => github.com/rogpeppe/go-internal upgrade + github.com/rs/xid => github.com/rs/xid upgrade + github.com/rs/zerolog => github.com/rs/zerolog upgrade + github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 upgrade + github.com/ruudk/golang-pdf417 => github.com/ruudk/golang-pdf417 upgrade + github.com/schollz/progressbar/v2 => github.com/schollz/progressbar/v2 upgrade + github.com/scylladb/go-reflectx => github.com/scylladb/go-reflectx upgrade + github.com/scylladb/gocqlx => github.com/scylladb/gocqlx upgrade + github.com/sergi/go-diff => github.com/sergi/go-diff upgrade + github.com/shopspring/decimal => github.com/shopspring/decimal upgrade + github.com/shurcooL/httpfs => github.com/shurcooL/httpfs upgrade + github.com/shurcooL/vfsgen => github.com/shurcooL/vfsgen upgrade + github.com/sirupsen/logrus => github.com/sirupsen/logrus upgrade + github.com/spf13/afero => github.com/spf13/afero upgrade + github.com/spf13/cobra => github.com/spf13/cobra upgrade + github.com/spf13/pflag => github.com/spf13/pflag upgrade + github.com/stoewer/go-strcase => github.com/stoewer/go-strcase upgrade + github.com/stretchr/objx => github.com/stretchr/objx upgrade + github.com/stretchr/testify => github.com/stretchr/testify upgrade + github.com/ugorji/go/codec => github.com/ugorji/go/codec upgrade + github.com/xeipuuv/gojsonpointer => github.com/xeipuuv/gojsonpointer upgrade + github.com/xeipuuv/gojsonreference => github.com/xeipuuv/gojsonreference upgrade + github.com/xeipuuv/gojsonschema => github.com/xeipuuv/gojsonschema upgrade + github.com/xlab/treeprint => github.com/xlab/treeprint upgrade + github.com/zeebo/assert => github.com/zeebo/assert upgrade + github.com/zeebo/xxh3 => github.com/zeebo/xxh3 upgrade + go.opencensus.io => go.opencensus.io upgrade + go.etcd.io/bbolt => go.etcd.io/bbolt upgrade go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.11.1 go.opentelemetry.io/otel/exporters/otlp/internal/retry => go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 @@ -283,64 +283,64 @@ replace ( go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.11.1 go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.33.0 go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.11.1 - go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp latest - go.starlark.net => go.starlark.net latest - go.uber.org/atomic => go.uber.org/atomic latest - go.uber.org/automaxprocs => go.uber.org/automaxprocs latest - go.uber.org/goleak => go.uber.org/goleak latest - go.uber.org/multierr => go.uber.org/multierr latest - go.uber.org/zap => go.uber.org/zap latest - gocloud.dev => gocloud.dev latest - golang.org/x/crypto => golang.org/x/crypto latest - golang.org/x/exp => golang.org/x/exp latest - golang.org/x/exp/typeparams => golang.org/x/exp/typeparams latest - golang.org/x/image => golang.org/x/image latest - golang.org/x/lint => golang.org/x/lint latest - golang.org/x/mobile => golang.org/x/mobile latest - golang.org/x/mod => golang.org/x/mod latest - golang.org/x/net => golang.org/x/net latest - golang.org/x/oauth2 => golang.org/x/oauth2 latest - golang.org/x/sync => golang.org/x/sync latest - golang.org/x/sys => golang.org/x/sys latest - golang.org/x/term => golang.org/x/term latest - golang.org/x/text => golang.org/x/text latest - golang.org/x/time => golang.org/x/time latest - golang.org/x/tools => golang.org/x/tools latest - golang.org/x/xerrors => golang.org/x/xerrors latest - gomodules.xyz/jsonpatch/v2 => gomodules.xyz/jsonpatch/v2 latest - gonum.org/v1/gonum => gonum.org/v1/gonum latest - gonum.org/v1/hdf5 => gonum.org/v1/hdf5 latest - gonum.org/v1/plot => gonum.org/v1/plot latest - google.golang.org/api => google.golang.org/api latest - google.golang.org/appengine => google.golang.org/appengine latest - google.golang.org/genproto => google.golang.org/genproto latest - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api latest - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc latest - google.golang.org/grpc => google.golang.org/grpc latest - google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc latest - google.golang.org/protobuf => google.golang.org/protobuf latest - gopkg.in/check.v1 => gopkg.in/check.v1 latest - gopkg.in/inconshreveable/log15.v2 => gopkg.in/inconshreveable/log15.v2 latest - gopkg.in/inf.v0 => gopkg.in/inf.v0 latest - gopkg.in/tomb.v1 => gopkg.in/tomb.v1 latest - gopkg.in/yaml.v2 => gopkg.in/yaml.v2 latest - gopkg.in/yaml.v3 => gopkg.in/yaml.v3 latest - honnef.co/go/tools => honnef.co/go/tools latest + go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp upgrade + go.starlark.net => go.starlark.net upgrade + go.uber.org/atomic => go.uber.org/atomic upgrade + go.uber.org/automaxprocs => go.uber.org/automaxprocs upgrade + go.uber.org/goleak => go.uber.org/goleak upgrade + go.uber.org/multierr => go.uber.org/multierr upgrade + go.uber.org/zap => go.uber.org/zap upgrade + gocloud.dev => gocloud.dev upgrade + golang.org/x/crypto => golang.org/x/crypto upgrade + golang.org/x/exp => golang.org/x/exp upgrade + golang.org/x/exp/typeparams => golang.org/x/exp/typeparams upgrade + golang.org/x/image => golang.org/x/image upgrade + golang.org/x/lint => golang.org/x/lint upgrade + golang.org/x/mobile => golang.org/x/mobile upgrade + golang.org/x/mod => golang.org/x/mod upgrade + golang.org/x/net => golang.org/x/net upgrade + golang.org/x/oauth2 => golang.org/x/oauth2 upgrade + golang.org/x/sync => golang.org/x/sync upgrade + golang.org/x/sys => golang.org/x/sys upgrade + golang.org/x/term => golang.org/x/term upgrade + golang.org/x/text => golang.org/x/text upgrade + golang.org/x/time => golang.org/x/time upgrade + golang.org/x/tools => golang.org/x/tools upgrade + golang.org/x/xerrors => golang.org/x/xerrors upgrade + gomodules.xyz/jsonpatch/v2 => gomodules.xyz/jsonpatch/v2 upgrade + gonum.org/v1/gonum => gonum.org/v1/gonum upgrade + gonum.org/v1/hdf5 => gonum.org/v1/hdf5 upgrade + gonum.org/v1/plot => gonum.org/v1/plot upgrade + google.golang.org/api => google.golang.org/api upgrade + google.golang.org/appengine => google.golang.org/appengine upgrade + google.golang.org/genproto => google.golang.org/genproto upgrade + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api upgrade + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc upgrade + google.golang.org/grpc => google.golang.org/grpc upgrade + google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc upgrade + google.golang.org/protobuf => google.golang.org/protobuf upgrade + gopkg.in/check.v1 => gopkg.in/check.v1 upgrade + gopkg.in/inconshreveable/log15.v2 => gopkg.in/inconshreveable/log15.v2 upgrade + gopkg.in/inf.v0 => gopkg.in/inf.v0 upgrade + gopkg.in/tomb.v1 => gopkg.in/tomb.v1 upgrade + gopkg.in/yaml.v2 => gopkg.in/yaml.v2 upgrade + gopkg.in/yaml.v3 => gopkg.in/yaml.v3 upgrade + honnef.co/go/tools => honnef.co/go/tools upgrade k8s.io/api => k8s.io/api v0.27.3 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.27.3 k8s.io/apimachinery => k8s.io/apimachinery v0.27.3 k8s.io/cli-runtime => k8s.io/cli-runtime v0.27.3 k8s.io/client-go => k8s.io/client-go v0.27.3 k8s.io/component-base => k8s.io/component-base v0.27.3 - k8s.io/klog/v2 => k8s.io/klog/v2 latest + k8s.io/klog/v2 => k8s.io/klog/v2 upgrade k8s.io/kube-openapi => k8s.io/kube-openapi 2546d827e515dca59571ec245eef2302e11018e1 k8s.io/kubernetes => k8s.io/kubernetes v0.27.3 k8s.io/metrics => k8s.io/metrics v0.27.3 - nhooyr.io/websocket => nhooyr.io/websocket latest - rsc.io/pdf => rsc.io/pdf latest + nhooyr.io/websocket => nhooyr.io/websocket upgrade + rsc.io/pdf => rsc.io/pdf upgrade sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.15.0 - sigs.k8s.io/json => sigs.k8s.io/json latest - sigs.k8s.io/kustomize => sigs.k8s.io/kustomize latest - sigs.k8s.io/structured-merge-diff/v4 => sigs.k8s.io/structured-merge-diff/v4 latest - sigs.k8s.io/yaml => sigs.k8s.io/yaml latest + sigs.k8s.io/json => sigs.k8s.io/json upgrade + sigs.k8s.io/kustomize => sigs.k8s.io/kustomize upgrade + sigs.k8s.io/structured-merge-diff/v4 => sigs.k8s.io/structured-merge-diff/v4 upgrade + sigs.k8s.io/yaml => sigs.k8s.io/yaml upgrade ) diff --git a/internal/config/corrector.go b/internal/config/corrector.go new file mode 100644 index 0000000000..3e1176c8bb --- /dev/null +++ b/internal/config/corrector.go @@ -0,0 +1,62 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package config providers configuration type and load configuration logic +package config + +// Corrector represents the index correction configurations. +type Corrector struct { + // AgentPort represent agent port number + AgentPort int `json:"agent_port" yaml:"agent_port"` + + // AgentName represent agents meta_name for service discovery + AgentName string `json:"agent_name" yaml:"agent_name"` + + // AgentNamespace represent agent namespace location + AgentNamespace string `json:"agent_namespace" yaml:"agent_namespace"` + + // AgentDNS represent agents dns A record for service discovery + AgentDNS string `json:"agent_dns" yaml:"agent_dns"` + + // NodeName represents node name + NodeName string `json:"node_name" yaml:"node_name"` + + // StreamConcurrency represent stream concurrency for StreamListObject rpc client + // this directly affects the memory usage of this job + StreamListConcurrency int `json:"stream_list_concurrency" yaml:"stream_list_concurrency"` + + // BboltAsyncWriteConcurrency represent concurrency for bbolt async write + BboltAsyncWriteConcurrency int `json:"bbolt_async_write_concurrency" yaml:"bbolt_async_write_concurrency"` + + // IndexReplica represent index replica count. This should be equal to the lb setting + IndexReplica int `json:"index_replica" yaml:"index_replica"` + + // Discoverer represent agent discoverer service configuration + Discoverer *DiscovererClient `json:"discoverer" yaml:"discoverer"` +} + +// Bind binds the actual data from the Indexer receiver field. +func (c *Corrector) Bind() *Corrector { + c.AgentName = GetActualValue(c.AgentName) + c.AgentNamespace = GetActualValue(c.AgentNamespace) + c.AgentDNS = GetActualValue(c.AgentDNS) + c.NodeName = GetActualValue(c.NodeName) + + if c.Discoverer != nil { + c.Discoverer = c.Discoverer.Bind() + } + return c +} diff --git a/internal/config/corrector_test.go b/internal/config/corrector_test.go new file mode 100644 index 0000000000..0391c4d3aa --- /dev/null +++ b/internal/config/corrector_test.go @@ -0,0 +1,137 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestCorrector_Bind(t *testing.T) { +// type fields struct { +// AgentPort int +// AgentName string +// AgentNamespace string +// AgentDNS string +// NodeName string +// StreamListConcurrency int +// BboltAsyncWriteConcurrency int +// IndexReplica int +// Discoverer *DiscovererClient +// } +// type want struct { +// want *Corrector +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, *Corrector) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got *Corrector) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// AgentPort:0, +// AgentName:"", +// AgentNamespace:"", +// AgentDNS:"", +// NodeName:"", +// StreamListConcurrency:0, +// BboltAsyncWriteConcurrency:0, +// IndexReplica:0, +// Discoverer:DiscovererClient{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// AgentPort:0, +// AgentName:"", +// AgentNamespace:"", +// AgentDNS:"", +// NodeName:"", +// StreamListConcurrency:0, +// BboltAsyncWriteConcurrency:0, +// IndexReplica:0, +// Discoverer:DiscovererClient{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &Corrector{ +// AgentPort: test.fields.AgentPort, +// AgentName: test.fields.AgentName, +// AgentNamespace: test.fields.AgentNamespace, +// AgentDNS: test.fields.AgentDNS, +// NodeName: test.fields.NodeName, +// StreamListConcurrency: test.fields.StreamListConcurrency, +// BboltAsyncWriteConcurrency: test.fields.BboltAsyncWriteConcurrency, +// IndexReplica: test.fields.IndexReplica, +// Discoverer: test.fields.Discoverer, +// } +// +// got := c.Bind() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/config/index_creation.go b/internal/config/index_creation.go new file mode 100644 index 0000000000..8024fc068e --- /dev/null +++ b/internal/config/index_creation.go @@ -0,0 +1,57 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// IndexCreation represents the configurations for index creation. +type IndexCreation struct { + // AgentPort represent agent port number + AgentPort int `json:"agent_port" yaml:"agent_port"` + + // AgentName represent agents meta_name for service discovery + AgentName string `json:"agent_name" yaml:"agent_name"` + + // AgentNamespace represent agent namespace location + AgentNamespace string `json:"agent_namespace" yaml:"agent_namespace"` + + // AgentDNS represent agents dns A record for service discovery + AgentDNS string `json:"agent_dns" yaml:"agent_dns"` + + // NodeName represents node name + NodeName string `json:"node_name" yaml:"node_name"` + + // Concurrency represents indexing concurrency. + Concurrency int `json:"concurrency" yaml:"concurrency"` + + // CreationPoolSize represents batch pool size for indexing. + CreationPoolSize uint32 `json:"creation_pool_size" yaml:"creation_pool_size"` + + // TargetAddrs represents indexing target addresses. + TargetAddrs []string `json:"target_addrs" yaml:"target_addrs"` + + // Discoverer represents agent discoverer service configuration. + Discoverer *DiscovererClient `json:"discoverer" yaml:"discoverer"` +} + +func (ic *IndexCreation) Bind() *IndexCreation { + ic.AgentName = GetActualValue(ic.AgentName) + ic.AgentNamespace = GetActualValue(ic.AgentNamespace) + ic.AgentDNS = GetActualValue(ic.AgentDNS) + ic.NodeName = GetActualValue(ic.NodeName) + ic.TargetAddrs = GetActualValues(ic.TargetAddrs) + + if ic.Discoverer != nil { + ic.Discoverer.Bind() + } + return ic +} diff --git a/internal/config/index_creation_test.go b/internal/config/index_creation_test.go new file mode 100644 index 0000000000..52d8e65ca2 --- /dev/null +++ b/internal/config/index_creation_test.go @@ -0,0 +1,137 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestIndexCreation_Bind(t *testing.T) { +// type fields struct { +// AgentPort int +// AgentName string +// AgentNamespace string +// AgentDNS string +// NodeName string +// Concurrency int +// CreationPoolSize uint32 +// TargetAddrs []string +// Discoverer *DiscovererClient +// } +// type want struct { +// want *IndexCreation +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, *IndexCreation) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got *IndexCreation) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// AgentPort:0, +// AgentName:"", +// AgentNamespace:"", +// AgentDNS:"", +// NodeName:"", +// Concurrency:0, +// CreationPoolSize:0, +// TargetAddrs:nil, +// Discoverer:DiscovererClient{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// AgentPort:0, +// AgentName:"", +// AgentNamespace:"", +// AgentDNS:"", +// NodeName:"", +// Concurrency:0, +// CreationPoolSize:0, +// TargetAddrs:nil, +// Discoverer:DiscovererClient{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// ic := &IndexCreation{ +// AgentPort: test.fields.AgentPort, +// AgentName: test.fields.AgentName, +// AgentNamespace: test.fields.AgentNamespace, +// AgentDNS: test.fields.AgentDNS, +// NodeName: test.fields.NodeName, +// Concurrency: test.fields.Concurrency, +// CreationPoolSize: test.fields.CreationPoolSize, +// TargetAddrs: test.fields.TargetAddrs, +// Discoverer: test.fields.Discoverer, +// } +// +// got := ic.Bind() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/config/index_save.go b/internal/config/index_save.go new file mode 100644 index 0000000000..290909e34a --- /dev/null +++ b/internal/config/index_save.go @@ -0,0 +1,54 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// IndexSave represents the configurations for index save. +type IndexSave struct { + // AgentPort represent agent port number. + AgentPort int `json:"agent_port" yaml:"agent_port"` + + // AgentName represent agents meta_name for service discovery. + AgentName string `json:"agent_name" yaml:"agent_name"` + + // AgentNamespace represent agent namespace location. + AgentNamespace string `json:"agent_namespace" yaml:"agent_namespace"` + + // AgentDNS represent agents dns A record for service discovery. + AgentDNS string `json:"agent_dns" yaml:"agent_dns"` + + // NodeName represents node name. + NodeName string `json:"node_name" yaml:"node_name"` + + // Concurrency represents indexing concurrency. + Concurrency int `json:"concurrency" yaml:"concurrency"` + + // TargetAddrs represents indexing target addresses. + TargetAddrs []string `json:"target_addrs" yaml:"target_addrs"` + + // Discoverer represents agent discoverer service configuration. + Discoverer *DiscovererClient `json:"discoverer" yaml:"discoverer"` +} + +func (is *IndexSave) Bind() *IndexSave { + is.AgentName = GetActualValue(is.AgentName) + is.AgentNamespace = GetActualValue(is.AgentNamespace) + is.AgentDNS = GetActualValue(is.AgentDNS) + is.NodeName = GetActualValue(is.NodeName) + is.TargetAddrs = GetActualValues(is.TargetAddrs) + + if is.Discoverer != nil { + is.Discoverer.Bind() + } + return is +} diff --git a/internal/config/index_save_test.go b/internal/config/index_save_test.go new file mode 100644 index 0000000000..f363c77b0e --- /dev/null +++ b/internal/config/index_save_test.go @@ -0,0 +1,133 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestIndexSave_Bind(t *testing.T) { +// type fields struct { +// AgentPort int +// AgentName string +// AgentNamespace string +// AgentDNS string +// NodeName string +// Concurrency int +// TargetAddrs []string +// Discoverer *DiscovererClient +// } +// type want struct { +// want *IndexSave +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, *IndexSave) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got *IndexSave) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// AgentPort:0, +// AgentName:"", +// AgentNamespace:"", +// AgentDNS:"", +// NodeName:"", +// Concurrency:0, +// TargetAddrs:nil, +// Discoverer:DiscovererClient{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// AgentPort:0, +// AgentName:"", +// AgentNamespace:"", +// AgentDNS:"", +// NodeName:"", +// Concurrency:0, +// TargetAddrs:nil, +// Discoverer:DiscovererClient{}, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// is := &IndexSave{ +// AgentPort: test.fields.AgentPort, +// AgentName: test.fields.AgentName, +// AgentNamespace: test.fields.AgentNamespace, +// AgentDNS: test.fields.AgentDNS, +// NodeName: test.fields.NodeName, +// Concurrency: test.fields.Concurrency, +// TargetAddrs: test.fields.TargetAddrs, +// Discoverer: test.fields.Discoverer, +// } +// +// got := is.Bind() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/internal/errors/corrector.go b/internal/errors/corrector.go new file mode 100644 index 0000000000..5391f3591a --- /dev/null +++ b/internal/errors/corrector.go @@ -0,0 +1,43 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package errors provides error types and function +package errors + +// ErrIndexReplicaOne represents an error that nothing to correct when index replica is 1. +var ErrIndexReplicaOne = New("nothing to correct when index replica is 1") + +// ErrAgentReplicaOne represents an error that nothing to correct when agent replica is 1. +var ErrAgentReplicaOne = New("nothing to correct when agent replica is 1") + +// ErrNoAvailableAgentToInsert represents an error that no available agent to insert replica. +var ErrNoAvailableAgentToInsert = New("no available agent to insert replica") + +// ErrFailedToCorrectReplicaNum represents an error that failed to correct replica number after correction process. +var ErrFailedToCorrectReplicaNum = New("failed to correct replica number after correction process") + +// ErrFailedToReceiveVectorFromStream represents an error that failed to receive vector from stream while index correction process. +var ErrFailedToReceiveVectorFromStream = New("failed to receive vector from stream") + +// ErrFailedToCheckConsistency represents an error that failed to check consistency process while index correction process. +var ErrFailedToCheckConsistency = func(err error) error { + return Wrap(err, "failed to check consistency while index correctioin process") +} + +// ErrStreamListObjectStreamFinishedUnexpectedly represents an error that StreamListObject finished not because of io.EOF. +var ErrStreamListObjectStreamFinishedUnexpectedly = func(err error) error { + return Wrap(err, "stream list object stream finished unexpectedly") +} diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 3fd58dc0ec..0e6e5b6b43 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -18,10 +18,12 @@ package errors import ( + "cmp" "errors" "fmt" "reflect" "runtime" + "slices" "strings" "github.com/vdaas/vald/internal/sync" @@ -279,6 +281,16 @@ func Join(errs ...error) error { return e } +func RemoveDuplicates(errs []error) []error { + if len(errs) < 2 { + return errs + } + slices.SortStableFunc(errs, func(l error, r error) int { + return cmp.Compare(l.Error(), r.Error()) + }) + return slices.CompactFunc(errs, Is) +} + type joinError struct { errs []error } diff --git a/internal/errors/errors_test.go b/internal/errors/errors_test.go index 8a6b70e747..32f7e99bef 100644 --- a/internal/errors/errors_test.go +++ b/internal/errors/errors_test.go @@ -1581,6 +1581,74 @@ func TestAs(t *testing.T) { } } +func TestRemoveDuplicates(t *testing.T) { + type args struct { + errs []error + } + tests := []struct { + name string + args args + want []error + }{ + { + name: "succeeds to remove duplicated errors", + args: args{ + errs: []error{ + New("same error1"), + New("same error1"), + New("same error2"), + New("same error2"), + New("same error2"), + New("same error3"), + }, + }, + want: []error{ + New("same error1"), + New("same error2"), + New("same error3"), + }, + }, + { + name: "single error remains the same", + args: args{ + errs: []error{ + New("same error"), + }, + }, + want: []error{ + New("same error"), + }, + }, + { + name: "empty errs remains the same", + args: args{ + errs: []error{}, + }, + want: []error{}, + }, + } + + equalErrs := func(errs1, errs2 []error) bool { + if len(errs1) != len(errs2) { + return false + } + for i := range errs1 { + if !Is(errs1[i], errs2[i]) { + return false + } + } + return true + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RemoveDuplicates(tt.args.errs); !equalErrs(got, tt.want) { + t.Errorf("removeDuplicatedErrs() = %v, want %v", got, tt.want) + } + }) + } +} + // NOT IMPLEMENTED BELOW // // func TestUnwrap(t *testing.T) { diff --git a/internal/errors/grpc.go b/internal/errors/grpc.go index b14c754fd5..a5675de8d6 100644 --- a/internal/errors/grpc.go +++ b/internal/errors/grpc.go @@ -70,4 +70,14 @@ var ( ErrInvalidProtoMessageType = func(v interface{}) error { return Errorf("failed to marshal/unmarshal proto message, message type is %T (missing vtprotobuf/protobuf helpers)", v) } + + // ErrServerStreamClientRecv represents a function to generate an error that the gRPC client couldn't receive from stream. + ErrServerStreamClientRecv = func(err error) error { + return Wrap(err, "gRPC client failed to receive from stream") + } + + // ErrServerStreamClientSend represents a function to generate an error that the gRPC server couldn't send to stream. + ErrServerStreamServerSend = func(err error) error { + return Wrap(err, "gRPC server failed to send to stream") + } ) diff --git a/internal/net/grpc/context.go b/internal/net/grpc/context.go index a95bfca402..90e5ee32a6 100644 --- a/internal/net/grpc/context.go +++ b/internal/net/grpc/context.go @@ -20,28 +20,30 @@ import ( type contextKey string -const grpcMethodContextKey contextKey = "grpc_method" +// GRPCMethodContextKey represents a context key for gRPC method. +// This is exported only for testing. +const GRPCMethodContextKey contextKey = "grpc_method" // WrapGRPCMethod returns a copy of parent in which the method associated with key (grpcMethodContextKey). func WrapGRPCMethod(ctx context.Context, method string) context.Context { m := FromGRPCMethod(ctx) if m == "" { - return context.WithValue(ctx, grpcMethodContextKey, method) + return context.WithValue(ctx, GRPCMethodContextKey, method) } if strings.HasSuffix(m, method) { return ctx } - return context.WithValue(ctx, grpcMethodContextKey, m+"/"+method) + return context.WithValue(ctx, GRPCMethodContextKey, m+"/"+method) } // WithGRPCMethod returns a copy of parent in which the method associated with key (grpcMethodContextKey). func WithGRPCMethod(ctx context.Context, method string) context.Context { - return context.WithValue(ctx, grpcMethodContextKey, method) + return context.WithValue(ctx, GRPCMethodContextKey, method) } // FromGRPCMethod returns the value associated with this context for key (grpcMethodContextKey). func FromGRPCMethod(ctx context.Context) string { - if v := ctx.Value(grpcMethodContextKey); v != nil { + if v := ctx.Value(GRPCMethodContextKey); v != nil { if method, ok := v.(string); ok { return method } diff --git a/internal/net/grpc/stream.go b/internal/net/grpc/stream.go index 4dc6e56de3..15ebbbd755 100644 --- a/internal/net/grpc/stream.go +++ b/internal/net/grpc/stream.go @@ -18,11 +18,9 @@ package grpc import ( - "cmp" "context" "fmt" "runtime" - "slices" "sync/atomic" "github.com/vdaas/vald/internal/errors" @@ -75,9 +73,7 @@ func BidirectionalStream[Q any, R any](ctx context.Context, stream ServerStream, errs = append(errs, err) emu.Unlock() } - removeDuplicates(errs, func(left, right error) int { - return cmp.Compare(left.Error(), right.Error()) - }) + errs := errors.RemoveDuplicates(errs) emu.Lock() err = errors.Join(errs...) emu.Unlock() @@ -229,11 +225,3 @@ func BidirectionalStreamClient(stream ClientStream, } }() } - -func removeDuplicates[S ~[]E, E comparable](x S, less func(left, right E) int) S { - if len(x) < 2 { - return x - } - slices.SortStableFunc(x, less) - return slices.Compact(x) -} diff --git a/internal/observability/metrics/index/job/correction/correction.go b/internal/observability/metrics/index/job/correction/correction.go new file mode 100644 index 0000000000..3027e5c5a2 --- /dev/null +++ b/internal/observability/metrics/index/job/correction/correction.go @@ -0,0 +1,121 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package correction + +import ( + "context" + + "github.com/vdaas/vald/internal/observability/metrics" + "github.com/vdaas/vald/pkg/index/job/correction/service" + "go.opentelemetry.io/otel/sdk/metric/aggregation" + "go.opentelemetry.io/otel/sdk/metric/view" +) + +const ( + checkedIndexCount = "index_job_correction_checked_index_count" + checkedIndexCountDesc = "The number of checked indexes while index correction job" + + correctedOldIndexCount = "index_job_correction_corrected_old_index_count" + correctedOldIndexCountDesc = "The number of corrected old indexes while index correction job" + + correctedReplicationCount = "index_job_correction_corrected_replication_count" + correctedReplicationCountDesc = "The number of operation happened to correct replication number while index correction job" +) + +type correctionMetrics struct { + correction service.Corrector +} + +func New(c service.Corrector) metrics.Metric { + return &correctionMetrics{ + correction: c, + } +} + +func (*correctionMetrics) View() ([]*metrics.View, error) { + checkedIndexCount, err := view.New( + view.MatchInstrumentName(checkedIndexCount), + view.WithSetDescription(checkedIndexCountDesc), + view.WithSetAggregation(aggregation.LastValue{}), + ) + if err != nil { + return nil, err + } + + oldIndexCount, err := view.New( + view.MatchInstrumentName(correctedOldIndexCount), + view.WithSetDescription(correctedOldIndexCountDesc), + view.WithSetAggregation(aggregation.LastValue{}), + ) + if err != nil { + return nil, err + } + + replicationCount, err := view.New( + view.MatchInstrumentName(correctedReplicationCount), + view.WithSetDescription(correctedReplicationCountDesc), + view.WithSetAggregation(aggregation.LastValue{}), + ) + if err != nil { + return nil, err + } + + return []*metrics.View{ + &checkedIndexCount, + &oldIndexCount, + &replicationCount, + }, nil +} + +func (c *correctionMetrics) Register(m metrics.Meter) error { + checkedIndexCount, err := m.AsyncInt64().Gauge( + checkedIndexCount, + metrics.WithDescription(checkedIndexCountDesc), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + oldIndexCount, err := m.AsyncInt64().Gauge( + correctedOldIndexCount, + metrics.WithDescription(correctedOldIndexCountDesc), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + replicationCount, err := m.AsyncInt64().Gauge( + correctedReplicationCount, + metrics.WithDescription(correctedReplicationCountDesc), + metrics.WithUnit(metrics.Dimensionless), + ) + if err != nil { + return err + } + + return m.RegisterCallback( + []metrics.AsynchronousInstrument{ + checkedIndexCount, + oldIndexCount, + replicationCount, + }, + func(ctx context.Context) { + checkedIndexCount.Observe(ctx, int64(c.correction.NumberOfCheckedIndex())) + oldIndexCount.Observe(ctx, int64(c.correction.NumberOfCorrectedOldIndex())) + replicationCount.Observe(ctx, int64(c.correction.NumberOfCorrectedReplication())) + }, + ) +} diff --git a/internal/servers/server/server.go b/internal/servers/server/server.go index 904e76d371..7d094906c5 100644 --- a/internal/servers/server/server.go +++ b/internal/servers/server/server.go @@ -28,6 +28,7 @@ import ( "time" "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net" "github.com/vdaas/vald/internal/net/control" @@ -128,6 +129,8 @@ type grpcKeepalive struct { permitWithoutStream bool } +// New returns Server implementation. +// skipcq: GO-R1005 func New(opts ...Option) (Server, error) { srv := new(server) @@ -253,6 +256,7 @@ func (s *server) Name() string { return s.name } +// skipcq: GO-R1005 func (s *server) ListenAndServe(ctx context.Context, ech chan<- error) (err error) { if !s.IsRunning() { s.mu.Lock() @@ -274,8 +278,9 @@ func (s *server) ListenAndServe(ctx context.Context, ech chan<- error) (err erro return s.network.String() }(), func() string { if s.network == net.UNIX { - if len(s.socketPath) == 0 { - s.socketPath = os.TempDir() + string(os.PathSeparator) + s.name + "." + strconv.Itoa(os.Getpid()) + ".sock" + if s.socketPath == "" { + sockFile := strings.Join([]string{s.name, strconv.Itoa(os.Getpid()), "sock"}, ".") + s.socketPath = file.Join(os.TempDir(), sockFile) } return s.socketPath } @@ -334,12 +339,12 @@ func (s *server) ListenAndServe(ctx context.Context, ech chan<- error) (err erro s.mu.RUnlock() log.Infof("%s server %s stopped", s.mode.String(), s.name) } - return nil })) } return nil } +// skipcq: GO-R1005 func (s *server) Shutdown(ctx context.Context) (rerr error) { if !s.IsRunning() { return nil @@ -386,7 +391,7 @@ func (s *server) Shutdown(ctx context.Context) (rerr error) { } } - if len(s.socketPath) != 0 { + if s.socketPath != "" { defer func() { err := os.RemoveAll(s.socketPath) if err != nil { diff --git a/internal/test/mock/client/discoverer_client_mock.go b/internal/test/mock/client/discoverer_client_mock.go new file mode 100644 index 0000000000..3b368eacbe --- /dev/null +++ b/internal/test/mock/client/discoverer_client_mock.go @@ -0,0 +1,38 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package client + +import ( + "context" + + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/net/grpc" +) + +// DiscovererClientMock is the mock for discoverer client. +type DiscovererClientMock struct { + discoverer.Client + GetAddrsFunc func(ctx context.Context) []string + GetClientFunc func() grpc.Client +} + +// GetAddrs calls the GetAddrsFunc object. +func (dc *DiscovererClientMock) GetAddrs(ctx context.Context) []string { + return dc.GetAddrsFunc(ctx) +} + +// GetClient calls GetClientFunc object. +func (dc *DiscovererClientMock) GetClient() grpc.Client { + return dc.GetClientFunc() +} diff --git a/internal/test/mock/grpc/grpc_client_mock.go b/internal/test/mock/grpc/grpc_client_mock.go new file mode 100644 index 0000000000..926bbf8c61 --- /dev/null +++ b/internal/test/mock/grpc/grpc_client_mock.go @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package grpc + +import ( + "context" + + "github.com/vdaas/vald/internal/net/grpc" +) + +// GRPCClientMock is the mock for gRPC client. +type GRPCClientMock struct { + grpc.Client + OrderedRangeConcurrentFunc func(ctx context.Context, + order []string, + concurrency int, + f func(ctx context.Context, + addr string, + conn *grpc.ClientConn, + copts ...grpc.CallOption) error) error +} + +// OrderedRangeConcurrent calls the OrderedRangeConcurrentFunc object. +func (gc *GRPCClientMock) OrderedRangeConcurrent(ctx context.Context, + order []string, + concurrency int, + f func(ctx context.Context, + addr string, + conn *grpc.ClientConn, + copts ...grpc.CallOption) error, +) error { + return gc.OrderedRangeConcurrentFunc(ctx, order, concurrency, f) +} diff --git a/internal/test/mock/grpc_testify_mock.go b/internal/test/mock/grpc_testify_mock.go index 5fcd6a41fa..9083d65554 100644 --- a/internal/test/mock/grpc_testify_mock.go +++ b/internal/test/mock/grpc_testify_mock.go @@ -15,9 +15,13 @@ package mock import ( "context" + "fmt" "github.com/stretchr/testify/mock" "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/backoff" + "github.com/vdaas/vald/internal/net/grpc/pool" + "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -63,3 +67,139 @@ func (losm *ListObjectStreamMock) Send(res *payload.Object_List_Response) error args := losm.Called(res) return args.Error(0) } + +type ClientInternal struct { + mock.Mock +} + +type ( + CallOption = grpc.CallOption + DialOption = pool.DialOption + ClientConn = pool.ClientConn +) + +func (c *ClientInternal) StartConnectionMonitor(ctx context.Context) (<-chan error, error) { + args := c.Called(ctx) + return args.Get(0).(<-chan error), args.Error(1) +} + +func (c *ClientInternal) Connect(ctx context.Context, addr string, dopts ...DialOption) (pool.Conn, error) { + args := c.Called(ctx, addr, dopts) + return args.Get(0).(pool.Conn), args.Error(1) +} + +func (c *ClientInternal) IsConnected(ctx context.Context, addr string) bool { + args := c.Called(ctx, addr) + return args.Bool(0) +} + +func (c *ClientInternal) Disconnect(ctx context.Context, addr string) error { + args := c.Called(ctx, addr) + return args.Error(0) +} + +func (c *ClientInternal) Range(ctx context.Context, + f func(ctx context.Context, + addr string, + conn *ClientConn, + copts ...CallOption) error, +) error { + args := c.Called(ctx, f) + return args.Error(0) +} + +func (c *ClientInternal) RangeConcurrent(ctx context.Context, + concurrency int, + f func(ctx context.Context, + addr string, + conn *ClientConn, + copts ...CallOption) error, +) error { + args := c.Called(ctx, concurrency, f) + return args.Error(0) +} + +func (c *ClientInternal) OrderedRange(ctx context.Context, + order []string, + f func(ctx context.Context, + addr string, + conn *ClientConn, + copts ...CallOption) error, +) error { + args := c.Called(ctx, order, f) + return args.Error(0) +} + +func (c *ClientInternal) OrderedRangeConcurrent(ctx context.Context, + order []string, + concurrency int, + f func(ctx context.Context, + addr string, + conn *ClientConn, + copts ...CallOption) error, +) error { + args := c.Called(ctx, order, concurrency, f) + return args.Error(0) +} + +func (c *ClientInternal) Do(ctx context.Context, addr string, + f func(ctx context.Context, + conn *ClientConn, + copts ...CallOption) (interface{}, error), +) (interface{}, error) { + args := c.Called(ctx, addr, f) + return args.Get(0), args.Error(1) +} + +func (c *ClientInternal) RoundRobin(ctx context.Context, f func(ctx context.Context, + conn *ClientConn, + copts ...CallOption) (interface{}, error), +) (interface{}, error) { + args := c.Called(ctx, f) + return args.Get(0), args.Error(1) +} + +func (c *ClientInternal) GetDialOption() []DialOption { + args := c.Called() + v, ok := args.Get(0).([]DialOption) + if !ok { + // panic here like testify mock does + panic(fmt.Sprintf("The provided arg(%v) is not type []DialOption", args.Get(0))) + } + return v +} + +func (c *ClientInternal) GetCallOption() []CallOption { + args := c.Called() + v, ok := args.Get(0).([]CallOption) + if !ok { + // panic here like testify mock does + panic(fmt.Sprintf("The provided arg(%v) is not type []CallOption", args.Get(0))) + } + return v +} + +func (c *ClientInternal) GetBackoff() backoff.Backoff { + args := c.Called() + v, ok := args.Get(0).(backoff.Backoff) + if !ok { + // panic here like testify mock does + panic(fmt.Sprintf("The provided arg(%v) is not type backoff.Backoff", args.Get(0))) + } + return v +} + +func (c *ClientInternal) ConnectedAddrs() []string { + args := c.Called() + v, ok := args.Get(0).([]string) + if !ok { + // panic here like testify mock does + panic(fmt.Sprintf("The provided arg(%v) is not type []string", args.Get(0))) + } + return v +} + +func (c *ClientInternal) Close(ctx context.Context) error { + args := c.Called(ctx) + return args.Error(0) +} diff --git a/k8s/index/job/correction/configmap.yaml b/k8s/index/job/correction/configmap.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/k8s/index/job/correction/configmap.yaml @@ -0,0 +1 @@ +--- diff --git a/k8s/index/job/correction/cronjob.yaml b/k8s/index/job/correction/cronjob.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/k8s/index/job/correction/cronjob.yaml @@ -0,0 +1 @@ +--- diff --git a/k8s/metrics/grafana/dashboards/09-vald-index-correction.yaml b/k8s/metrics/grafana/dashboards/09-vald-index-correction.yaml new file mode 100644 index 0000000000..031f16864a --- /dev/null +++ b/k8s/metrics/grafana/dashboards/09-vald-index-correction.yaml @@ -0,0 +1,1410 @@ +--- +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-vald-index-correction +data: + vald-index-correction.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^vald_version$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Vald Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^go_version$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}, \"go_version\", \"v$1\", \"go_version\", \"([^v].*)\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Go Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^go_os$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Go OS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 100 + }, + { + "color": "#d44a3a", + "value": 300 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 8, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(kube_pod_info{namespace=\"$Namespace\", pod=~\"$ReplicaSet.*\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Pods ($ReplicaSet)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10000000000 + }, + { + "color": "#d44a3a", + "value": 1000000000000 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$Namespace\", container=\"$ReplicaSet\", image!=\"\"})", + "format": "time_series", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Total memory working set ($ReplicaSet)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^git_commit$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Git Commit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 3 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^build_time$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "app_version_info{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Build at", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(irate(container_cpu_usage_seconds_total{namespace=\"$Namespace\", container=\"$ReplicaSet\", pod=~\"$PodName\", image!=\"\"}[$interval])) by (pod) and on() count(kube_job_created{job_name=\"$ReplicaSet\"}) >= 1", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 100 + }, + { + "color": "#d44a3a", + "value": 300 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 27, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "index_job_correction_checked_index_count{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "checked index count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 100 + }, + { + "color": "#d44a3a", + "value": 300 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 28, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "index_job_correction_corrected_old_index_count{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "corrected old index count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 100 + }, + { + "color": "#d44a3a", + "value": 300 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 29, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "index_job_correction_corrected_replication_count{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "corrected replication count", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{namespace=\"$Namespace\", container=\"$ReplicaSet\", pod=~\"$PodName\", image!=\"\"}) by (pod) and on() count(kube_job_created{job_name=\"$ReplicaSet\"}) >= 1", + "interval": "", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory working set", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:154", + "format": "decbytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:155", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "goroutine_count{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_pod=~\"$PodName\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{target_pod}}", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "goroutine count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 26, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "increase(gc_count{exported_kubernetes_namespace=\"$Namespace\", kubernetes_name=~\"$ReplicaSet\", target_node=~\".+\"}[$interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{target_pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "GC count /s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(kube_pod_info, namespace)", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "Namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "vald-index-correction", + "value": "vald-index-correction" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_version_info{server_name=~\"index correction job\"}, kubernetes_name)", + "hide": 0, + "includeAll": false, + "label": "name", + "multi": false, + "name": "ReplicaSet", + "options": [], + "query": { + "query": "label_values(app_version_info{server_name=~\"index correction job\"}, kubernetes_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_version_info{server_name=~\"index correction job\", kubernetes_name=\"$ReplicaSet\"}, target_pod)", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "PodName", + "options": [], + "query": { + "query": "label_values(app_version_info{server_name=~\"index correction job\", kubernetes_name=\"$ReplicaSet\"}, target_pod)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": true, + "text": "1m", + "value": "1m" + }, + "hide": 0, + "label": "interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "2m", + "value": "2m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,2m,5m,10m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Vald Index Correction", + "uid": "a8fc1362-e1b8-419f-91de-5205c1c82476", + "version": 1, + "weekStart": "" + } diff --git a/k8s/metrics/grafana/deployment.yaml b/k8s/metrics/grafana/deployment.yaml index 44abf79f98..19fcf14070 100644 --- a/k8s/metrics/grafana/deployment.yaml +++ b/k8s/metrics/grafana/deployment.yaml @@ -50,6 +50,8 @@ spec: mountPath: /var/lib/grafana/dashboards-vald/07 - name: grafana-dashboards-vald-lb-gateway mountPath: /var/lib/grafana/dashboards-vald/08 + - name: grafana-dashboards-vald-index-correction + mountPath: /var/lib/grafana/dashboards-vald/09 volumes: - name: grafana-datasource-provider configMap: @@ -83,3 +85,7 @@ spec: configMap: defaultMode: 420 name: grafana-dashboards-vald-lb-gateway + - name: grafana-dashboards-vald-index-correction + configMap: + defaultMode: 420 + name: grafana-dashboards-vald-index-correction diff --git a/k8s/operator/helm/clusterrole.yaml b/k8s/operator/helm/clusterrole.yaml index 5137df5e71..ab151aaa8f 100644 --- a/k8s/operator/helm/clusterrole.yaml +++ b/k8s/operator/helm/clusterrole.yaml @@ -175,3 +175,15 @@ rules: - get - patch - update + - apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/k8s/operator/helm/crds/valdrelease.yaml b/k8s/operator/helm/crds/valdrelease.yaml index 46c151ab94..3854881577 100644 --- a/k8s/operator/helm/crds/valdrelease.yaml +++ b/k8s/operator/helm/crds/valdrelease.yaml @@ -6068,6 +6068,971 @@ spec: annotations: type: object x-kubernetes-preserve-unknown-fields: true + corrector: + type: object + properties: + agent_namespace: + type: string + bbolt_async_write_concurrency: + type: integer + minimum: 1 + discoverer: + type: object + properties: + agent_client_options: + type: object + properties: + addrs: + type: array + items: + type: string + backoff: + type: object + properties: + backoff_factor: + type: number + backoff_time_limit: + type: string + enable_error_log: + type: boolean + initial_duration: + type: string + jitter_limit: + type: string + maximum_duration: + type: string + retry_count: + type: integer + call_option: + type: object + x-kubernetes-preserve-unknown-fields: true + circuit_breaker: + type: object + properties: + closed_error_rate: + type: number + closed_refresh_timeout: + type: string + half_open_error_rate: + type: number + min_samples: + type: integer + open_timeout: + type: string + connection_pool: + type: object + properties: + enable_dns_resolver: + type: boolean + enable_rebalance: + type: boolean + old_conn_close_duration: + type: string + rebalance_duration: + type: string + size: + type: integer + dial_option: + type: object + properties: + backoff_base_delay: + type: string + backoff_jitter: + type: number + backoff_max_delay: + type: string + backoff_multiplier: + type: number + enable_backoff: + type: boolean + initial_connection_window_size: + type: integer + initial_window_size: + type: integer + insecure: + type: boolean + interceptors: + type: array + items: + type: string + enum: + - TraceInterceptor + keepalive: + type: object + properties: + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_msg_size: + type: integer + min_connection_timeout: + type: string + net: + type: object + properties: + dialer: + type: object + properties: + dual_stack_enabled: + type: boolean + keepalive: + type: string + timeout: + type: string + dns: + type: object + properties: + cache_enabled: + type: boolean + cache_expiration: + type: string + refresh_duration: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + read_buffer_size: + type: integer + timeout: + type: string + write_buffer_size: + type: integer + health_check_duration: + type: string + max_recv_msg_size: + type: integer + max_retry_rpc_buffer_size: + type: integer + max_send_msg_size: + type: integer + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + wait_for_ready: + type: boolean + client: + type: object + properties: + addrs: + type: array + items: + type: string + backoff: + type: object + properties: + backoff_factor: + type: number + backoff_time_limit: + type: string + enable_error_log: + type: boolean + initial_duration: + type: string + jitter_limit: + type: string + maximum_duration: + type: string + retry_count: + type: integer + call_option: + type: object + x-kubernetes-preserve-unknown-fields: true + circuit_breaker: + type: object + properties: + closed_error_rate: + type: number + closed_refresh_timeout: + type: string + half_open_error_rate: + type: number + min_samples: + type: integer + open_timeout: + type: string + connection_pool: + type: object + properties: + enable_dns_resolver: + type: boolean + enable_rebalance: + type: boolean + old_conn_close_duration: + type: string + rebalance_duration: + type: string + size: + type: integer + dial_option: + type: object + properties: + backoff_base_delay: + type: string + backoff_jitter: + type: number + backoff_max_delay: + type: string + backoff_multiplier: + type: number + enable_backoff: + type: boolean + initial_connection_window_size: + type: integer + initial_window_size: + type: integer + insecure: + type: boolean + interceptors: + type: array + items: + type: string + enum: + - TraceInterceptor + keepalive: + type: object + properties: + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_msg_size: + type: integer + min_connection_timeout: + type: string + net: + type: object + properties: + dialer: + type: object + properties: + dual_stack_enabled: + type: boolean + keepalive: + type: string + timeout: + type: string + dns: + type: object + properties: + cache_enabled: + type: boolean + cache_expiration: + type: string + refresh_duration: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + read_buffer_size: + type: integer + timeout: + type: string + write_buffer_size: + type: integer + health_check_duration: + type: string + max_recv_msg_size: + type: integer + max_retry_rpc_buffer_size: + type: integer + max_send_msg_size: + type: integer + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + wait_for_ready: + type: boolean + duration: + type: string + enabled: + type: boolean + env: + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true + image: + type: object + properties: + pullPolicy: + type: string + enum: + - Always + - Never + - IfNotPresent + repository: + type: string + tag: + type: string + name: + type: string + node_name: + type: string + observability: + type: object + properties: + enabled: + type: boolean + metrics: + type: object + properties: + enable_cgo: + type: boolean + enable_goroutine: + type: boolean + enable_memory: + type: boolean + enable_version_info: + type: boolean + version_info_labels: + type: array + items: + type: string + enum: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - cgo_enabled + - ngt_version + - build_cpu_info_flags + otlp: + type: object + properties: + attribute: + type: object + properties: + namespace: + type: string + node_name: + type: string + pod_name: + type: string + service_name: + type: string + collector_endpoint: + type: string + metrics_export_interval: + type: string + metrics_export_timeout: + type: string + trace_batch_timeout: + type: string + trace_export_timeout: + type: string + trace_max_export_batch_size: + type: integer + trace_max_queue_size: + type: integer + trace: + type: object + properties: + enabled: + type: boolean + schedule: + type: string + server_config: + type: object + properties: + full_shutdown_duration: + type: string + healths: + type: object + properties: + liveness: + type: object + properties: + enabled: + type: boolean + host: + type: string + livenessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + readiness: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + readinessProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + startup: + type: object + properties: + enabled: + type: boolean + port: + type: integer + maximum: 65535 + minimum: 0 + startupProbe: + type: object + properties: + failureThreshold: + type: integer + httpGet: + type: object + properties: + path: + type: string + port: + type: string + scheme: + type: string + initialDelaySeconds: + type: integer + periodSeconds: + type: integer + successThreshold: + type: integer + timeoutSeconds: + type: integer + metrics: + type: object + properties: + pprof: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + servers: + type: object + properties: + grpc: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + grpc: + type: object + properties: + bidirectional_stream_concurrency: + type: integer + connection_timeout: + type: string + enable_reflection: + type: boolean + header_table_size: + type: integer + initial_conn_window_size: + type: integer + initial_window_size: + type: integer + interceptors: + type: array + items: + type: string + enum: + - RecoverInterceptor + - AccessLogInterceptor + - TraceInterceptor + - MetricInterceptor + keepalive: + type: object + properties: + max_conn_age: + type: string + max_conn_age_grace: + type: string + max_conn_idle: + type: string + min_time: + type: string + permit_without_stream: + type: boolean + time: + type: string + timeout: + type: string + max_header_list_size: + type: integer + max_receive_message_size: + type: integer + max_send_message_size: + type: integer + read_buffer_size: + type: integer + write_buffer_size: + type: integer + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + restart: + type: boolean + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + rest: + type: object + properties: + enabled: + type: boolean + host: + type: string + port: + type: integer + maximum: 65535 + minimum: 0 + server: + type: object + properties: + http: + type: object + properties: + handler_timeout: + type: string + idle_timeout: + type: string + read_header_timeout: + type: string + read_timeout: + type: string + shutdown_duration: + type: string + write_timeout: + type: string + mode: + type: string + network: + type: string + enum: + - tcp + - tcp4 + - tcp6 + - udp + - udp4 + - udp6 + - unix + - unixgram + - unixpacket + probe_wait_time: + type: string + socket_option: + type: object + properties: + ip_recover_destination_addr: + type: boolean + ip_transparent: + type: boolean + reuse_addr: + type: boolean + reuse_port: + type: boolean + tcp_cork: + type: boolean + tcp_defer_accept: + type: boolean + tcp_fast_open: + type: boolean + tcp_no_delay: + type: boolean + tcp_quick_ack: + type: boolean + socket_path: + type: string + servicePort: + type: integer + maximum: 65535 + minimum: 0 + tls: + type: object + properties: + ca: + type: string + cert: + type: string + enabled: + type: boolean + insecure_skip_verify: + type: boolean + key: + type: string + startingDeadlineSeconds: + type: integer + stream_list_concurrency: + type: integer + minimum: 1 + suspend: + type: boolean + ttlSecondsAfterFinished: + type: integer + version: + type: string + pattern: ^v[0-9]+\.[0-9]+\.[0-9]$ enabled: type: boolean env: diff --git a/pkg/gateway/lb/handler/grpc/handler.go b/pkg/gateway/lb/handler/grpc/handler.go index 27f5bf8c90..0554a9c2cb 100644 --- a/pkg/gateway/lb/handler/grpc/handler.go +++ b/pkg/gateway/lb/handler/grpc/handler.go @@ -20,6 +20,7 @@ package grpc import ( "context" "fmt" + "io" "slices" "strconv" "sync/atomic" @@ -2907,7 +2908,7 @@ func (s *server) getObject(ctx context.Context, uuid string) (vec *payload.Objec ech <- s.gateway.BroadCast(ctx, func(ctx context.Context, target string, vc vald.Client, copts ...grpc.CallOption) error { sctx, sspan := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "BroadCast/"+target), apiName+"/getObject/BroadCast/"+target) defer func() { - if span != nil { + if sspan != nil { sspan.End() } }() @@ -3134,3 +3135,90 @@ func (s *server) StreamGetObject(stream vald.Object_StreamGetObjectServer) (err } return nil } + +func (s *server) StreamListObject(req *payload.Object_List_Request, stream vald.Object_StreamListObjectServer) error { + ctx, span := trace.StartSpan(grpc.WithGRPCMethod(stream.Context(), vald.PackageName+"."+vald.ObjectRPCServiceName+"/"+vald.StreamListObjectRPCName), apiName+"/"+vald.StreamListObjectRPCName) + defer func() { + if span != nil { + span.End() + } + }() + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var rmu, smu sync.Mutex + err := s.gateway.BroadCast(ctx, func(ctx context.Context, target string, vc vald.Client, copts ...grpc.CallOption) error { + ctx, sspan := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "BroadCast/"+target), apiName+"/"+vald.StreamListObjectRPCName+"/"+target) + defer func() { + if sspan != nil { + sspan.End() + } + }() + + client, err := vc.StreamListObject(ctx, req, copts...) + if err != nil { + log.Errorf("failed to get StreamListObject client for agent(%s): %v", target, err) + return err + } + + eg, ctx := errgroup.WithContext(ctx) + ectx, ecancel := context.WithCancel(ctx) + defer ecancel() + eg.SetLimit(s.streamConcurrency) + + for { + select { + case <-ectx.Done(): + var err error + if !errors.Is(ctx.Err(), context.Canceled) { + err = errors.Join(err, ctx.Err()) + } + if egerr := eg.Wait(); err != nil { + err = errors.Join(err, egerr) + } + return err + default: + eg.Go(safety.RecoverFunc(func() error { + rmu.Lock() + res, err := client.Recv() + rmu.Unlock() + if err != nil { + if errors.Is(err, io.EOF) { + ecancel() + return nil + } + return errors.ErrServerStreamClientRecv(err) + } + + vec := res.GetVector() + if vec == nil { + st := res.GetStatus() + log.Warnf("received empty vector: code %v: details %v: message %v", + st.GetCode(), + st.GetDetails(), + st.GetMessage(), + ) + return nil + } + + smu.Lock() + err = stream.Send(res) + smu.Unlock() + if err != nil { + if sspan != nil { + st, msg, err := status.ParseError(err, codes.Internal, "failed to parse StreamListObject send gRPC error response") + sspan.RecordError(err) + sspan.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) + sspan.SetStatus(trace.StatusError, err.Error()) + } + return errors.ErrServerStreamServerSend(err) + } + + return nil + })) + } + } + }) + return err +} diff --git a/pkg/gateway/lb/service/gateway.go b/pkg/gateway/lb/service/gateway.go index a58cd8a5f3..7d32364df0 100644 --- a/pkg/gateway/lb/service/gateway.go +++ b/pkg/gateway/lb/service/gateway.go @@ -36,9 +36,9 @@ type Gateway interface { GetAgentCount(ctx context.Context) int Addrs(ctx context.Context) []string DoMulti(ctx context.Context, num int, - f func(ctx context.Context, tgt string, ac vald.Client, copts ...grpc.CallOption) error) error + f func(ctx context.Context, target string, ac vald.Client, copts ...grpc.CallOption) error) error BroadCast(ctx context.Context, - f func(ctx context.Context, tgt string, ac vald.Client, copts ...grpc.CallOption) error) error + f func(ctx context.Context, target string, ac vald.Client, copts ...grpc.CallOption) error) error } type gateway struct { diff --git a/pkg/index/job/correction/config/config.go b/pkg/index/job/correction/config/config.go new file mode 100644 index 0000000000..aff43f3657 --- /dev/null +++ b/pkg/index/job/correction/config/config.go @@ -0,0 +1,76 @@ +// +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Package setting stores all server application settings +package config + +import ( + "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" +) + +type GlobalConfig = config.GlobalConfig + +// Data represents a application setting data content (config.yaml). +// In K8s environment, this configuration is stored in K8s ConfigMap. +type Data struct { + config.GlobalConfig `json:",inline" yaml:",inline"` + + // Server represent all server configurations + Server *config.Servers `json:"server_config" yaml:"server_config"` + + // Observability represent observability configurations + Observability *config.Observability `json:"observability" yaml:"observability"` + + // Indexer represent agent auto indexing service configuration + Corrector *config.Corrector `json:"corrector" yaml:"corrector"` +} + +func NewConfig(path string) (cfg *Data, err error) { + cfg = new(Data) + + err = config.Read(path, &cfg) + + if err != nil { + return nil, err + } + + if cfg != nil { + cfg.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Server != nil { + cfg.Server = cfg.Server.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Observability != nil { + cfg.Observability = cfg.Observability.Bind() + } else { + cfg.Observability = new(config.Observability).Bind() + } + + if cfg.Corrector != nil { + cfg.Corrector = cfg.Corrector.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + return cfg, nil +} diff --git a/pkg/index/job/correction/config/config_test.go b/pkg/index/job/correction/config/config_test.go new file mode 100644 index 0000000000..0cf6858bf7 --- /dev/null +++ b/pkg/index/job/correction/config/config_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestNewConfig(t *testing.T) { +// type args struct { +// path string +// } +// type want struct { +// wantCfg *Data +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, *Data, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCfg *Data, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCfg, w.wantCfg) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCfg, w.wantCfg) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotCfg, err := NewConfig(test.args.path) +// if err := checkFunc(test.want, gotCfg, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/correction/service/corrector.go b/pkg/index/job/correction/service/corrector.go new file mode 100644 index 0000000000..f345268005 --- /dev/null +++ b/pkg/index/job/correction/service/corrector.go @@ -0,0 +1,638 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "cmp" + "context" + "fmt" + "io" + "os" + "reflect" + "slices" + "sync/atomic" + "time" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/db/kvs/bbolt" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync" + "github.com/vdaas/vald/internal/sync/errgroup" +) + +type contextTimeKey string + +const ( + insertMethod = "core.v1.Vald/Insert" + updateMethod = "core.v1.Vald/Update" + deleteMethod = "core.v1.Vald/Delete" + correctionStartTimeKey contextTimeKey = "correctionStartTimeKey" +) + +type Corrector interface { + Start(ctx context.Context) error + StartClient(ctx context.Context) (<-chan error, error) + PreStop(ctx context.Context) error + // For metrics + NumberOfCheckedIndex() uint64 + NumberOfCorrectedOldIndex() uint64 + NumberOfCorrectedReplication() uint64 +} + +type correct struct { + discoverer discoverer.Client + agentAddrs []string + indexInfos sync.Map[string, *payload.Info_Index_Count] + uuidsCount uint32 + uncommittedUUIDsCount uint32 + checkedID bbolt.Bbolt + checkedIndexCount atomic.Uint64 + correctedOldIndexCount atomic.Uint64 + correctedReplicationCount atomic.Uint64 + + indexReplica int + streamListConcurrency int + bboltAsyncWriteConcurrency int +} + +const filemode = 0o600 + +func New(opts ...Option) (_ Corrector, err error) { + c := new(correct) + for _, opt := range append(defaultOpts, opts...) { + if err := opt(c); err != nil { + oerr := errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + e := &errors.ErrCriticalOption{} + if errors.As(oerr, &e) { + log.Error(err) + return nil, oerr + } + log.Warn(oerr) + } + } + if err := c.bboltInit(); err != nil { + return nil, err + } + return c, nil +} + +func (c *correct) bboltInit() error { + dpath := file.Join(os.TempDir(), "bbolt") + err := file.MkdirAll(dpath, os.ModePerm) + if err != nil { + return err + } + + dbfile := file.Join(dpath, "checkedid.db") + c.checkedID, err = bbolt.New(dbfile, "", os.FileMode(filemode)) + if err != nil { + return err + } + return nil +} + +func (c *correct) StartClient(ctx context.Context) (<-chan error, error) { + return c.discoverer.Start(ctx) +} + +func (c *correct) Start(ctx context.Context) error { + // set current time to context + ctx = embedTime(ctx) + + // addrs is sorted by the memory usage of each agent(descending order) + // this is decending because it's supposed to be used for index manager to decide + // which pod to make a create index rpc(higher memory, first to commit) + c.agentAddrs = c.discoverer.GetAddrs(ctx) + if len(c.agentAddrs) <= 1 { + log.Warnf("target agent (%v) found, but there must be more than two agents for correction to happen", c.agentAddrs) + return errors.ErrAgentReplicaOne + } + log.Debugf("target agent addrs: %v", c.agentAddrs) + + if err := c.loadInfos(ctx); err != nil { + return err + } + c.indexInfos.Range(func(addr string, info *payload.Info_Index_Count) bool { + log.Infof("index info: addr(%s), stored(%d), uncommitted(%d)", addr, info.GetStored(), info.GetUncommitted()) + return true + }) + + log.Info("starting correction with bbolt disk cache...") + if err := c.correct(ctx); err != nil { + return err + } + log.Info("correction finished successfully") + + return nil +} + +func (c *correct) PreStop(_ context.Context) error { + log.Info("removing persistent cache files...") + return c.checkedID.Close(true) +} + +func (c *correct) NumberOfCheckedIndex() uint64 { + return c.checkedIndexCount.Load() +} + +func (c *correct) NumberOfCorrectedOldIndex() uint64 { + return c.correctedOldIndexCount.Load() +} + +func (c *correct) NumberOfCorrectedReplication() uint64 { + return c.correctedReplicationCount.Load() +} + +// skipcq: GO-R1005 +func (c *correct) correct(ctx context.Context) (err error) { + // leftAgentAddrs is the agents' addr that hasn't been corrected yet. + // This is used to know which agents possibly have the same index as the target replica. + // We can say this because, thanks to caching, there is no way that the target replica is + // in the agent that has already been corrected. + + // Vector with time after this should not be processed + correctionStartTime, err := correctionStartTime(ctx) + if err != nil { + log.Errorf("cannot determine correction start time: %w", err) + return err + } + + curTargetAgent := 0 + jobErrs := make([]error, 0, c.streamListConcurrency) + if err := c.discoverer.GetClient().OrderedRange(ctx, c.agentAddrs, + func(ctx context.Context, addr string, conn *grpc.ClientConn, copts ...grpc.CallOption) (err error) { + // current address is the leftAgentAddrs[0] because this is OrderedRange and + // leftAgentAddrs is copied from c.agentAddrs + defer func() { + if err != nil { + // catch the err that happened in this scope using named return err + jobErrs = append(jobErrs, err) + } + curTargetAgent++ + }() + + // context and errgroup for stream.Recv and correction + sctx, scancel := context.WithCancel(ctx) + defer scancel() + seg, sctx := errgroup.WithContext(sctx) + seg.SetLimit(c.streamListConcurrency) + + // errgroup for bbolt AsyncSet + bolteg, ctx := errgroup.WithContext(ctx) + bolteg.SetLimit(c.bboltAsyncWriteConcurrency) + + log.Infof("starting correction for agent %s, stream concurrency: %d, bbolt concurrency: %d", addr, c.streamListConcurrency, c.bboltAsyncWriteConcurrency) + + vc := vald.NewValdClient(conn) + stream, err := vc.StreamListObject(ctx, &payload.Object_List_Request{}) + if err != nil { + return err + } + + var mu sync.Mutex + // The number of items to be received in advance is not known in advance. + // This is because there is a possibility of new items being inserted during processing. + for { + select { + case <-sctx.Done(): + if !errors.Is(sctx.Err(), context.Canceled) { + log.Errorf("context done unexpectedly: %v", sctx.Err()) + } + + // Finalize + err = seg.Wait() + if err != nil { + log.Errorf("err group returned error: %v", err) + } + + berr := bolteg.Wait() + if berr != nil { + log.Errorf("bbolt err group returned error: %v", err) + err = errors.Join(err, berr) + } else { + log.Info("bbolt all batch finished") + } + + log.Infof("correction finished for agent %s", addr) + return err + + default: + seg.Go(safety.RecoverFunc(func() error { + mu.Lock() + // As long as we don't stream.Recv() from the stream, we do not consume the memory of the message. + // So by limiting the number of this errgroup.Go instances, we can limit the memory usage + // https://github.com/grpc/grpc-go/blob/33f9fa2e6e5bcf4cf8fe45133e23779ae6e43f6c/rpc_util.go#L795 + res, err := stream.Recv() + mu.Unlock() + + if err != nil { + if errors.Is(err, io.EOF) { + scancel() + return nil + } + return errors.ErrStreamListObjectStreamFinishedUnexpectedly(err) + } + + vec := res.GetVector() + if vec == nil { + st := res.GetStatus() + log.Error(st.GetCode(), st.GetMessage(), st.GetDetails()) + return errors.ErrFailedToReceiveVectorFromStream + } + + // skip if the vector is inserted after correction start + if vec.GetTimestamp() > correctionStartTime.UnixNano() { + log.Debugf("timestamp of vector(id: %s, timestamp: %v) is newer than correction start time(%v). skipping...", + vec.GetId(), + vec.GetTimestamp(), + correctionStartTime.UnixNano(), + ) + return nil + } + + // check if the index is already checked + id := vec.GetId() + _, ok, err := c.checkedID.Get([]byte(id)) + if err != nil { + log.Errorf("failed to perform Get from bbolt but still try to finish processing without cache: %v", err) + } + if ok { + // already checked index + return nil + } + + if err := c.checkConsistency( + ctx, + &vectorReplica{ + addr: addr, + vec: vec, + }, + curTargetAgent, + ); err != nil { + return errors.ErrFailedToCheckConsistency(err) + } + + // now this id is checked so set it to the disk cache + c.checkedID.AsyncSet(bolteg, []byte(id), nil) + c.checkedIndexCount.Add(1) + + return nil + })) + } + } + }, + ); err != nil { + // This only happnes when ErrGRPCClientConnNotFound is returned. + // In other cases, OrderedRange continues processing, so jobErrrs is used to keep track of the error status of correction. + return err + } + + jobErrs = errors.RemoveDuplicates(jobErrs) + return errors.Join(jobErrs...) +} + +type vectorReplica struct { + addr string + vec *payload.Object_Vector +} + +// Validate len(addrs) >= 2 before calling this function +func (c *correct) checkConsistency(ctx context.Context, targetReplica *vectorReplica, targetAgentIdx int) error { + // leftAgentAddrs is the agents' addr that hasn't been corrected yet. + leftAgentAddrs := c.agentAddrs[targetAgentIdx+1:] + + // Vector with time after this should not be processed + correctionStartTime, err := correctionStartTime(ctx) + if err != nil { + log.Errorf("cannot determine correction start time: %w", err) + return err + } + + foundReplicas := make([]*vectorReplica, 0, len(c.agentAddrs)) + var mu sync.Mutex + if err := c.discoverer.GetClient().OrderedRangeConcurrent(ctx, leftAgentAddrs, len(leftAgentAddrs), + func(ctx context.Context, addr string, conn *grpc.ClientConn, copts ...grpc.CallOption) error { + vec, err := vald.NewValdClient(conn).GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: targetReplica.vec.GetId(), + }, + }) + if err != nil { + if st, ok := status.FromError(err); !ok { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return err + } else if st.Code() == codes.NotFound { + // when replica of agent > index replica, this happens + return nil + } else { + log.Errorf("failed to GetObject with unexpected error. code: %v, message: %s", st.Code(), st.Message()) + return err + } + } + + // skip if the vector is inserted after correction start + if vec.GetTimestamp() > correctionStartTime.UnixNano() { + log.Debugf("timestamp of vector(id: %s, timestamp: %v) is newer than correction start time(%v). skipping...", + vec.GetId(), + vec.GetTimestamp(), + correctionStartTime.UnixNano(), + ) + return nil + } + + mu.Lock() + foundReplicas = append(foundReplicas, &vectorReplica{ + addr: addr, + vec: vec, + }) + mu.Unlock() + + return nil + }, + ); err != nil { + return err + } + + // check timestamps + if err := c.correctTimestamp(ctx, targetReplica, foundReplicas); err != nil { + return fmt.Errorf("failed to fix timestamp: %w", err) + } + + // check replica number + if err := c.correctReplica(ctx, targetReplica, foundReplicas); err != nil { + return fmt.Errorf("failed to fix index replica: %w", err) + } + + return nil +} + +func (c *correct) correctTimestamp(ctx context.Context, targetReplica *vectorReplica, foundReplicas []*vectorReplica) error { + if len(foundReplicas) == 0 { + // no replica found. nothing to do about timestamp + return nil + } + + // skipcq: CRT-D0001 + allReplicas := append(foundReplicas, targetReplica) + + // sort by timestamp + slices.SortFunc(allReplicas, func(i, j *vectorReplica) int { + // largest timestamp means the latest + return cmp.Compare(j.vec.GetTimestamp(), i.vec.GetTimestamp()) + }) + + latest := allReplicas[0] + latestTS := latest.vec.GetTimestamp() + for _, replica := range allReplicas { + if replica.vec.GetTimestamp() == latestTS { + // no inconsistency + continue + } + + // udate the vector with the new one + log.Infof("timestamp inconsistency detected with vector(id: %s, timestamp: %v). updating with the latest vector(id: %s, timestamp: %v)", + replica.vec.GetId(), + replica.vec.GetTimestamp(), + latest.vec.GetId(), + latest.vec.GetTimestamp(), + ) + c.correctedOldIndexCount.Add(1) + if err := c.updateObject(ctx, replica.addr, latest.vec); err != nil { + return err + } + } + + return nil +} + +// correctReplica corrects the number of replicas of the target vector. +// skipcq: GO-R1005 +func (c *correct) correctReplica( + ctx context.Context, + targetReplica *vectorReplica, + foundReplicas []*vectorReplica, +) error { + // diff < 0 means there is less replica than the correct number + existReplica := len(foundReplicas) + 1 + diff := existReplica - c.indexReplica + if diff == 0 { + // replica number is correct + return nil + } + + // availableAddrs = c.agentAddrs - foundReplicas - targetReplica.addr + availableAddrs := make([]string, 0, len(c.agentAddrs)) + for _, addr := range c.agentAddrs { + if addr == targetReplica.addr { + continue + } + if slices.ContainsFunc(foundReplicas, func(replica *vectorReplica) bool { + return replica.addr == addr + }) { + continue + } + availableAddrs = append(availableAddrs, addr) + } + + // when there are less replicas than the correct number, add the extra replicas + if diff < 0 { + log.Infof("replica shortage of vector %s. inserting to other agents...", targetReplica.vec.GetId()) + c.correctedReplicationCount.Add(1) + if len(availableAddrs) == 0 { + return errors.ErrNoAvailableAgentToInsert + } + + // inserting with the reverse order of availableAddrs since the last agent has the lowest memory usage + for i := len(availableAddrs) - 1; i >= 0 && diff < 0; i-- { + addr := availableAddrs[i] + log.Infof("inserting replica to %s", addr) + if err := c.insertObject(ctx, addr, targetReplica.vec); err != nil { + log.Errorf("failed to insert object to agent(%s): %v", addr, err) + continue + } + diff++ + } + + if diff < 0 { + return errors.ErrFailedToCorrectReplicaNum + } + + return nil + } + + // when there are more replicas than the correct number, delete the extra replicas + log.Infof("replica oversupply of vector %s. deleting...", + targetReplica.vec.GetId()) + c.correctedReplicationCount.Add(1) + // delete from myself + if err := c.deleteObject(ctx, targetReplica.addr, targetReplica.vec); err != nil { + log.Errorf("failed to delete object from agent(%s): %v", targetReplica.addr, err) + } else { + diff-- + } + + // delte from others if there's more to delete + for _, replica := range foundReplicas { + if diff == 0 { + break + } + if err := c.deleteObject(ctx, replica.addr, replica.vec); err != nil { + log.Errorf("failed to delete object from agent(%s): %v", replica.addr, err) + continue + } + diff-- + } + + if diff > 0 { + return errors.ErrFailedToCorrectReplicaNum + } + + return nil +} + +func (c *correct) updateObject(ctx context.Context, addr string, vector *payload.Object_Vector) error { + res, err := c.discoverer.GetClient(). + Do(grpc.WithGRPCMethod(ctx, updateMethod), addr, func(ctx context.Context, conn *grpc.ClientConn, copts ...grpc.CallOption) (interface{}, error) { + // TODO: use UpdateTimestamp when it's implemented because here we just want to update only the timestamp but not the vector + return vald.NewUpdateClient(conn).Update(ctx, &payload.Update_Request{ + Vector: vector, + // TODO: this should be deleted after Config.Timestamp deprecation + Config: &payload.Update_Config{ + // TODO: Decrementing because it's gonna be incremented befor being pushed + // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation + // so we should consider refactoring vqueue. + Timestamp: vector.GetTimestamp() - 1, + }, + }, copts...) + }) + if err != nil { + return err + } + + if v, ok := res.(*payload.Object_Location); ok { + log.Infof("vector successfully updated. address: %s, uuid: %v", addr, v.GetUuid()) + } + + return nil +} + +func (c *correct) insertObject(ctx context.Context, addr string, vector *payload.Object_Vector) error { + res, err := c.discoverer.GetClient(). + Do(grpc.WithGRPCMethod(ctx, insertMethod), addr, func(ctx context.Context, conn *grpc.ClientConn, copts ...grpc.CallOption) (interface{}, error) { + return vald.NewInsertClient(conn).Insert(ctx, &payload.Insert_Request{ + Vector: vector, + // TODO: this should be deleted after Config.Timestamp deprecation + Config: &payload.Insert_Config{ + Timestamp: vector.GetTimestamp(), + }, + }, copts...) + }) + if err != nil { + return err + } + + if v, ok := res.(*payload.Object_Location); ok { + log.Infof("vector successfully inserted. address: %s, uuid: %v", addr, v.GetUuid()) + } + + return nil +} + +func (c *correct) deleteObject(ctx context.Context, addr string, vector *payload.Object_Vector) error { + res, err := c.discoverer.GetClient(). + Do(grpc.WithGRPCMethod(ctx, deleteMethod), addr, func(ctx context.Context, conn *grpc.ClientConn, copts ...grpc.CallOption) (interface{}, error) { + return vald.NewRemoveClient(conn).Remove(ctx, &payload.Remove_Request{ + Id: &payload.Object_ID{ + Id: vector.GetId(), + }, + }, copts...) + }) + if err != nil { + return err + } + + if v, ok := res.(*payload.Object_Location); ok { + log.Infof("vector successfully deleted. address: %s, uuid: %v", addr, v.GetUuid()) + } + + return nil +} + +func (c *correct) loadInfos(ctx context.Context) (err error) { + var u, ucu uint32 + var infoMap sync.Map[string, *payload.Info_Index_Count] + err = c.discoverer.GetClient().RangeConcurrent(ctx, len(c.discoverer.GetAddrs(ctx)), + func(ctx context.Context, + addr string, conn *grpc.ClientConn, copts ...grpc.CallOption, + ) (err error) { + select { + case <-ctx.Done(): + return nil + default: + info, err := agent.NewAgentClient(conn).IndexInfo(ctx, new(payload.Empty), copts...) + if err != nil { + log.Warnf("an error occurred while calling IndexInfo of %s: %s", addr, err) + return nil + } + infoMap.Store(addr, info) + atomic.AddUint32(&u, info.GetStored()) + atomic.AddUint32(&ucu, info.GetUncommitted()) + } + return nil + }) + if err != nil { + return err + } + atomic.StoreUint32(&c.uuidsCount, atomic.LoadUint32(&u)) + atomic.StoreUint32(&c.uncommittedUUIDsCount, atomic.LoadUint32(&ucu)) + c.indexInfos.Range(func(addr string, _ *payload.Info_Index_Count) bool { + info, ok := infoMap.Load(addr) + if !ok { + c.indexInfos.Delete(addr) + } + c.indexInfos.Store(addr, info) + infoMap.Delete(addr) + return true + }) + infoMap.Range(func(addr string, info *payload.Info_Index_Count) bool { + c.indexInfos.Store(addr, info) + return true + }) + return nil +} + +func embedTime(ctx context.Context) context.Context { + v := ctx.Value(correctionStartTimeKey) + if _, ok := v.(time.Time); ok { + return ctx + } + return context.WithValue(ctx, correctionStartTimeKey, time.Now()) +} + +func correctionStartTime(ctx context.Context) (time.Time, error) { + v := ctx.Value(correctionStartTimeKey) + if t, ok := v.(time.Time); ok { + return t, nil + } + return time.Time{}, fmt.Errorf("timeKey is not embedded in context") +} diff --git a/pkg/index/job/correction/service/corrector_test.go b/pkg/index/job/correction/service/corrector_test.go new file mode 100644 index 0000000000..a249d59f60 --- /dev/null +++ b/pkg/index/job/correction/service/corrector_test.go @@ -0,0 +1,1387 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "context" + "testing" + + tmock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/test/mock" +) + +type mockDiscovererClient struct { + client mock.ClientInternal +} + +func (*mockDiscovererClient) Start(context.Context) (<-chan error, error) { + return nil, nil +} + +func (*mockDiscovererClient) GetAddrs(context.Context) []string { + return nil +} + +func (m *mockDiscovererClient) GetClient() grpc.Client { + return &m.client +} + +func Test_correct_correctTimestamp(t *testing.T) { + t.Parallel() + + // This mock just returns nil and record args inside + m := mockDiscovererClient{} + m.client.On("Do", tmock.Anything, tmock.Anything, tmock.Anything).Return(nil, nil) + c := &correct{ + discoverer: &m, + } + + type args struct { + target *vectorReplica + found []*vectorReplica + } + + type want struct { + addrs []string + err error + } + + type test struct { + name string + args args + want want + } + + tests := []test{ + { + name: "nothing happens when no replica is found", + args: args{ + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + Timestamp: 100, + }, + }, + found: []*vectorReplica{}, + }, + want: want{ + addrs: nil, + err: nil, + }, + }, + { + name: "updates one found vec when found vecs are older than target", + args: args{ + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + Timestamp: 100, + }, + }, + found: []*vectorReplica{ + { + addr: "found", + vec: &payload.Object_Vector{ + Id: "found", + Timestamp: 99, + }, + }, + }, + }, + want: want{ + addrs: []string{"found"}, + err: nil, + }, + }, + { + name: "updates multiple found vecs when found vecs are older than target", + args: args{ + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + Timestamp: 100, + }, + }, + found: []*vectorReplica{ + { + addr: "found1", + vec: &payload.Object_Vector{ + Id: "found", + Timestamp: 99, + }, + }, + { + addr: "found2", + vec: &payload.Object_Vector{ + Id: "found", + Timestamp: 98, + }, + }, + }, + }, + want: want{ + addrs: []string{"found1", "found2"}, + err: nil, + }, + }, + { + name: "updates target vec when found vecs are newer than target", + args: args{ + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + Timestamp: 0, + }, + }, + found: []*vectorReplica{ + { + addr: "found1", + vec: &payload.Object_Vector{ + Id: "found", + Timestamp: 99, + }, + }, + }, + }, + want: want{ + addrs: []string{"target"}, + err: nil, + }, + }, + { + name: "updates target vec and one of found vecs with the latest found vec", + args: args{ + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + Timestamp: 0, + }, + }, + found: []*vectorReplica{ + { + addr: "found1", + vec: &payload.Object_Vector{ + Id: "found", + Timestamp: 99, + }, + }, + { + addr: "latest", + vec: &payload.Object_Vector{ + Id: "found", + Timestamp: 100, + }, + }, + }, + }, + want: want{ + addrs: []string{"target", "found1"}, + err: nil, + }, + }, + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + err := c.correctTimestamp(context.Background(), test.args.target, test.args.found) + require.Equal(tt, test.want.err, err) + + for _, addr := range test.want.addrs { + // check if the agents which need to be corrected are called + // checking calling parameter, like timestamp, is impossible because its inside of the function arg + m.client.AssertCalled(tt, "Do", tmock.Anything, addr, tmock.Anything) + } + }) + } +} + +func Test_correct_correctReplica(t *testing.T) { + t.Parallel() + + // This mock just returns nil and record args inside + m := mockDiscovererClient{} + m.client.On("Do", tmock.Anything, tmock.Anything, tmock.Anything).Return(nil, nil) + + type args struct { + indexReplica int + target *vectorReplica + found []*vectorReplica + availableAddrs []string + } + + type addrMethod struct { + addr string + method string + } + + type want struct { + addrMethods []addrMethod + err error + } + + type test struct { + name string + args args + want want + } + + tests := []test{ + { + name: "nothing happens when replica number sutisfies", + args: args{ + indexReplica: 2, + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + }, + }, + found: []*vectorReplica{ + { + addr: "found", + vec: &payload.Object_Vector{ + Id: "found", + }, + }, + }, + availableAddrs: []string{}, + }, + want: want{ + addrMethods: nil, + err: nil, + }, + }, + { + name: "insert replica when replica number is not enough", + args: args{ + indexReplica: 2, + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + }, + }, + found: []*vectorReplica{}, + availableAddrs: []string{"available"}, + }, + want: want{ + addrMethods: []addrMethod{ + { + addr: "available", + method: insertMethod, + }, + }, + err: nil, + }, + }, + { + name: "insert replica to the agent with most memory available", + args: args{ + indexReplica: 2, + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + }, + }, + found: []*vectorReplica{}, + // this is supposed to be sorted by memory usage with descending order + availableAddrs: []string{"most memory used", "second memory used"}, + }, + want: want{ + addrMethods: []addrMethod{ + { + addr: "second memory used", + method: insertMethod, + }, + }, + err: nil, + }, + }, + { + name: "delete replica from myself when replica number is too much by one", + args: args{ + indexReplica: 2, + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + }, + }, + found: []*vectorReplica{ + { + addr: "found1", + }, + { + addr: "found2", + }, + }, + availableAddrs: []string{}, + }, + want: want{ + addrMethods: []addrMethod{ + { + addr: "target", + method: deleteMethod, + }, + }, + err: nil, + }, + }, + { + name: "delete replica from myself and most memory used agent when replica number is too much by more than one", + args: args{ + indexReplica: 2, + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + }, + }, + found: []*vectorReplica{ + { + addr: "found1", + }, + { + addr: "found2", + }, + { + addr: "found3", + }, + }, + availableAddrs: []string{}, + }, + want: want{ + addrMethods: []addrMethod{ + { + addr: "target", + method: deleteMethod, + }, + { + addr: "found1", + method: deleteMethod, + }, + }, + err: nil, + }, + }, + { + name: "return ErrNoAvailableAgentToInsert when availableAddrs is empty when insertion required", + args: args{ + indexReplica: 2, + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + }, + }, + found: []*vectorReplica{}, + availableAddrs: []string{}, + }, + want: want{ + addrMethods: nil, + err: errors.ErrNoAvailableAgentToInsert, + }, + }, + { + name: "return ErrFailedToCorrectReplicaNum when there is not enough number of availableAddrs", + args: args{ + indexReplica: 3, + target: &vectorReplica{ + addr: "target", + vec: &payload.Object_Vector{ + Id: "target", + }, + }, + found: []*vectorReplica{}, + availableAddrs: []string{"available"}, + }, + want: want{ + addrMethods: nil, + err: errors.ErrFailedToCorrectReplicaNum, + }, + }, + } + + for _, tc := range tests { + test := tc + c := &correct{ + discoverer: &m, + indexReplica: test.args.indexReplica, + } + + // agentAddrs = availableAddrs + target.addr + found.addr + // skipcq: CRT-D0001 + c.agentAddrs = append(test.args.availableAddrs, test.args.target.addr) + for _, found := range test.args.found { + c.agentAddrs = append(c.agentAddrs, found.addr) + } + + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + err := c.correctReplica(context.Background(), test.args.target, test.args.found) + if test.want.err != nil { + require.ErrorIs(t, test.want.err, err) + } + + for _, am := range test.want.addrMethods { + // check if the agents which need to be corrected are called with the required method + // checking calling parameter, like timestamp, is impossible because its inside of the function arg + m.client.AssertCalled(tt, "Do", tmock.MatchedBy(func(ctx context.Context) bool { + method := ctx.Value(grpc.GRPCMethodContextKey) + val, ok := method.(string) + if !ok { + return false + } + return val == am.method + }), am.addr, tmock.Anything) + } + }) + } +} + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// opts []Option +// } +// type want struct { +// want Corrector +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Corrector, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Corrector, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, err := New(test.args.opts...) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_correct_StartClient(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// discoverer discoverer.Client +// agentAddrs []string +// indexInfos sync.Map[string, *payload.Info_Index_Count] +// uuidsCount uint32 +// uncommittedUUIDsCount uint32 +// checkedID bbolt.Bbolt +// checkedIndexCount atomic.Uint64 +// correctedOldIndexCount atomic.Uint64 +// correctedReplicationCount atomic.Uint64 +// indexReplica int +// streamListConcurrency int +// bboltAsyncWriteConcurrency int +// } +// type want struct { +// want <-chan error +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, <-chan error, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got <-chan error, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &correct{ +// discoverer: test.fields.discoverer, +// agentAddrs: test.fields.agentAddrs, +// indexInfos: test.fields.indexInfos, +// uuidsCount: test.fields.uuidsCount, +// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount, +// checkedID: test.fields.checkedID, +// checkedIndexCount: test.fields.checkedIndexCount, +// correctedOldIndexCount: test.fields.correctedOldIndexCount, +// correctedReplicationCount: test.fields.correctedReplicationCount, +// indexReplica: test.fields.indexReplica, +// streamListConcurrency: test.fields.streamListConcurrency, +// bboltAsyncWriteConcurrency: test.fields.bboltAsyncWriteConcurrency, +// } +// +// got, err := c.StartClient(test.args.ctx) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_correct_Start(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// discoverer discoverer.Client +// agentAddrs []string +// indexInfos sync.Map[string, *payload.Info_Index_Count] +// uuidsCount uint32 +// uncommittedUUIDsCount uint32 +// checkedID bbolt.Bbolt +// checkedIndexCount atomic.Uint64 +// correctedOldIndexCount atomic.Uint64 +// correctedReplicationCount atomic.Uint64 +// indexReplica int +// streamListConcurrency int +// bboltAsyncWriteConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &correct{ +// discoverer: test.fields.discoverer, +// agentAddrs: test.fields.agentAddrs, +// indexInfos: test.fields.indexInfos, +// uuidsCount: test.fields.uuidsCount, +// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount, +// checkedID: test.fields.checkedID, +// checkedIndexCount: test.fields.checkedIndexCount, +// correctedOldIndexCount: test.fields.correctedOldIndexCount, +// correctedReplicationCount: test.fields.correctedReplicationCount, +// indexReplica: test.fields.indexReplica, +// streamListConcurrency: test.fields.streamListConcurrency, +// bboltAsyncWriteConcurrency: test.fields.bboltAsyncWriteConcurrency, +// } +// +// err := c.Start(test.args.ctx) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_correct_PreStop(t *testing.T) { +// type args struct { +// in0 context.Context +// } +// type fields struct { +// discoverer discoverer.Client +// agentAddrs []string +// indexInfos sync.Map[string, *payload.Info_Index_Count] +// uuidsCount uint32 +// uncommittedUUIDsCount uint32 +// checkedID bbolt.Bbolt +// checkedIndexCount atomic.Uint64 +// correctedOldIndexCount atomic.Uint64 +// correctedReplicationCount atomic.Uint64 +// indexReplica int +// streamListConcurrency int +// bboltAsyncWriteConcurrency int +// } +// type want struct { +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// in0:nil, +// }, +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// in0:nil, +// }, +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &correct{ +// discoverer: test.fields.discoverer, +// agentAddrs: test.fields.agentAddrs, +// indexInfos: test.fields.indexInfos, +// uuidsCount: test.fields.uuidsCount, +// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount, +// checkedID: test.fields.checkedID, +// checkedIndexCount: test.fields.checkedIndexCount, +// correctedOldIndexCount: test.fields.correctedOldIndexCount, +// correctedReplicationCount: test.fields.correctedReplicationCount, +// indexReplica: test.fields.indexReplica, +// streamListConcurrency: test.fields.streamListConcurrency, +// bboltAsyncWriteConcurrency: test.fields.bboltAsyncWriteConcurrency, +// } +// +// err := c.PreStop(test.args.in0) +// if err := checkFunc(test.want, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_correct_NumberOfCheckedIndex(t *testing.T) { +// type fields struct { +// discoverer discoverer.Client +// agentAddrs []string +// indexInfos sync.Map[string, *payload.Info_Index_Count] +// uuidsCount uint32 +// uncommittedUUIDsCount uint32 +// checkedID bbolt.Bbolt +// checkedIndexCount atomic.Uint64 +// correctedOldIndexCount atomic.Uint64 +// correctedReplicationCount atomic.Uint64 +// indexReplica int +// streamListConcurrency int +// bboltAsyncWriteConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &correct{ +// discoverer: test.fields.discoverer, +// agentAddrs: test.fields.agentAddrs, +// indexInfos: test.fields.indexInfos, +// uuidsCount: test.fields.uuidsCount, +// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount, +// checkedID: test.fields.checkedID, +// checkedIndexCount: test.fields.checkedIndexCount, +// correctedOldIndexCount: test.fields.correctedOldIndexCount, +// correctedReplicationCount: test.fields.correctedReplicationCount, +// indexReplica: test.fields.indexReplica, +// streamListConcurrency: test.fields.streamListConcurrency, +// bboltAsyncWriteConcurrency: test.fields.bboltAsyncWriteConcurrency, +// } +// +// got := c.NumberOfCheckedIndex() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_correct_NumberOfCorrectedOldIndex(t *testing.T) { +// type fields struct { +// discoverer discoverer.Client +// agentAddrs []string +// indexInfos sync.Map[string, *payload.Info_Index_Count] +// uuidsCount uint32 +// uncommittedUUIDsCount uint32 +// checkedID bbolt.Bbolt +// checkedIndexCount atomic.Uint64 +// correctedOldIndexCount atomic.Uint64 +// correctedReplicationCount atomic.Uint64 +// indexReplica int +// streamListConcurrency int +// bboltAsyncWriteConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &correct{ +// discoverer: test.fields.discoverer, +// agentAddrs: test.fields.agentAddrs, +// indexInfos: test.fields.indexInfos, +// uuidsCount: test.fields.uuidsCount, +// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount, +// checkedID: test.fields.checkedID, +// checkedIndexCount: test.fields.checkedIndexCount, +// correctedOldIndexCount: test.fields.correctedOldIndexCount, +// correctedReplicationCount: test.fields.correctedReplicationCount, +// indexReplica: test.fields.indexReplica, +// streamListConcurrency: test.fields.streamListConcurrency, +// bboltAsyncWriteConcurrency: test.fields.bboltAsyncWriteConcurrency, +// } +// +// got := c.NumberOfCorrectedOldIndex() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_correct_NumberOfCorrectedReplication(t *testing.T) { +// type fields struct { +// discoverer discoverer.Client +// agentAddrs []string +// indexInfos sync.Map[string, *payload.Info_Index_Count] +// uuidsCount uint32 +// uncommittedUUIDsCount uint32 +// checkedID bbolt.Bbolt +// checkedIndexCount atomic.Uint64 +// correctedOldIndexCount atomic.Uint64 +// correctedReplicationCount atomic.Uint64 +// indexReplica int +// streamListConcurrency int +// bboltAsyncWriteConcurrency int +// } +// type want struct { +// want uint64 +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, uint64) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, got uint64) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// discoverer:nil, +// agentAddrs:nil, +// indexInfos:nil, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// checkedID:nil, +// checkedIndexCount:nil, +// correctedOldIndexCount:nil, +// correctedReplicationCount:nil, +// indexReplica:0, +// streamListConcurrency:0, +// bboltAsyncWriteConcurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// c := &correct{ +// discoverer: test.fields.discoverer, +// agentAddrs: test.fields.agentAddrs, +// indexInfos: test.fields.indexInfos, +// uuidsCount: test.fields.uuidsCount, +// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount, +// checkedID: test.fields.checkedID, +// checkedIndexCount: test.fields.checkedIndexCount, +// correctedOldIndexCount: test.fields.correctedOldIndexCount, +// correctedReplicationCount: test.fields.correctedReplicationCount, +// indexReplica: test.fields.indexReplica, +// streamListConcurrency: test.fields.streamListConcurrency, +// bboltAsyncWriteConcurrency: test.fields.bboltAsyncWriteConcurrency, +// } +// +// got := c.NumberOfCorrectedReplication() +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/correction/service/options.go b/pkg/index/job/correction/service/options.go new file mode 100644 index 0000000000..3d1ec633a7 --- /dev/null +++ b/pkg/index/job/correction/service/options.go @@ -0,0 +1,71 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/errors" +) + +// Option represents the functional option for index corrector. +type Option func(*correct) error + +var defaultOpts = []Option{ + WithStreamListConcurrency(200), //nolint:gomnd + WithBboltAsyncWriteConcurrency(2048), //nolint:gomnd +} + +// WithIndexReplica returns Option that sets index replica. +func WithIndexReplica(num int) Option { + return func(c *correct) error { + if num <= 1 { + return errors.NewErrCriticalOption("indexReplica", num, errors.ErrIndexReplicaOne) + } + c.indexReplica = num + return nil + } +} + +// WithDiscoverer returns Option that sets discoverer client. +func WithDiscoverer(client discoverer.Client) Option { + return func(c *correct) error { + if client == nil { + return errors.NewErrCriticalOption("discoverer", client) + } + c.discoverer = client + return nil + } +} + +// WithStreamListConcurrency returns Option that sets concurrency for StreamList field value. +func WithStreamListConcurrency(num int) Option { + return func(c *correct) error { + if num <= 0 { + return errors.NewErrInvalidOption("streamListConcurrency", num) + } + c.streamListConcurrency = num + return nil + } +} + +// WithBboltAsyncWriteConcurrency returns Option that sets concurrency for kvs async write. +func WithBboltAsyncWriteConcurrency(num int) Option { + return func(c *correct) error { + if num <= 0 { + return errors.NewErrInvalidOption("bboltAsyncWriteConcurrency", num) + } + c.bboltAsyncWriteConcurrency = num + return nil + } +} diff --git a/pkg/index/job/correction/service/options_test.go b/pkg/index/job/correction/service/options_test.go new file mode 100644 index 0000000000..ed1a312363 --- /dev/null +++ b/pkg/index/job/correction/service/options_test.go @@ -0,0 +1,360 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +// NOT IMPLEMENTED BELOW +// +// func TestWithIndexReplica(t *testing.T) { +// type args struct { +// num int +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithIndexReplica(test.args.num) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithDiscoverer(t *testing.T) { +// type args struct { +// client discoverer.Client +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// client:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// client:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithDiscoverer(test.args.client) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithStreamListConcurrency(t *testing.T) { +// type args struct { +// num int +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithStreamListConcurrency(test.args.num) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithBboltAsyncWriteConcurrency(t *testing.T) { +// type args struct { +// num int +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithBboltAsyncWriteConcurrency(test.args.num) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/correction/usecase/corrector.go b/pkg/index/job/correction/usecase/corrector.go new file mode 100644 index 0000000000..110e71d4a6 --- /dev/null +++ b/pkg/index/job/correction/usecase/corrector.go @@ -0,0 +1,226 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +import ( + "context" + "os" + "syscall" + "time" + + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + iconf "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/interceptor/server/recover" + "github.com/vdaas/vald/internal/observability" + "github.com/vdaas/vald/internal/observability/metrics/index/job/correction" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" + "github.com/vdaas/vald/pkg/index/job/correction/config" + "github.com/vdaas/vald/pkg/index/job/correction/service" +) + +type run struct { + eg errgroup.Group + cfg *config.Data + observability observability.Observability + server starter.Server + corrector service.Corrector +} + +func New(cfg *config.Data) (r runner.Runner, err error) { + eg := errgroup.Get() + + dOpts, err := cfg.Corrector.Discoverer.Client.Opts() + if err != nil { + return nil, err + } + // skipcq: CRT-D0001 + dOpts = append(dOpts, grpc.WithErrGroup(eg)) + + acOpts, err := cfg.Corrector.Discoverer.AgentClientOptions.Opts() + if err != nil { + return nil, err + } + // skipcq: CRT-D0001 + acOpts = append(acOpts, grpc.WithErrGroup(eg)) + + // Construct discoverer + discoverer, err := discoverer.New( + discoverer.WithAutoConnect(true), + discoverer.WithName(cfg.Corrector.AgentName), + discoverer.WithNamespace(cfg.Corrector.AgentNamespace), + discoverer.WithPort(cfg.Corrector.AgentPort), + discoverer.WithServiceDNSARecord(cfg.Corrector.AgentDNS), + discoverer.WithDiscovererClient(grpc.New(dOpts...)), + discoverer.WithDiscoverDuration(cfg.Corrector.Discoverer.Duration), + discoverer.WithOptions(acOpts...), + discoverer.WithNodeName(cfg.Corrector.NodeName), + discoverer.WithOnDiscoverFunc(func(ctx context.Context, c discoverer.Client, addrs []string) error { + last := len(addrs) - 1 + for i := 0; i < len(addrs)/2; i++ { + addrs[i], addrs[last-i] = addrs[last-i], addrs[i] + } + return nil + }), + ) + if err != nil { + return nil, err + } + + grpcServerOptions := []server.Option{ + server.WithGRPCOption( + grpc.ChainUnaryInterceptor(recover.RecoverInterceptor()), + grpc.ChainStreamInterceptor(recover.RecoverStreamInterceptor()), + ), + } + + // For health check and metrics + srv, err := starter.New(starter.WithConfig(cfg.Server), + starter.WithGRPC(func(sc *iconf.Server) []server.Option { + return grpcServerOptions + }), + ) + if err != nil { + return nil, err + } + + corrector, err := service.New( + service.WithDiscoverer(discoverer), + service.WithIndexReplica(cfg.Corrector.IndexReplica), + service.WithBboltAsyncWriteConcurrency(cfg.Corrector.BboltAsyncWriteConcurrency), + service.WithStreamListConcurrency(cfg.Corrector.StreamListConcurrency), + ) + if err != nil { + return nil, err + } + + var obs observability.Observability + if cfg.Observability.Enabled { + obs, err = observability.NewWithConfig( + cfg.Observability, + correction.New(corrector), + ) + if err != nil { + return nil, err + } + } + + return &run{ + eg: eg, + cfg: cfg, + observability: obs, + server: srv, + corrector: corrector, + }, nil +} + +func (r *run) PreStart(ctx context.Context) error { + if r.observability != nil { + return r.observability.PreStart(ctx) + } + return nil +} + +func (r *run) Start(ctx context.Context) (<-chan error, error) { + log.Info("starting servers") + ech := make(chan error, 3) //nolint:gomnd + var oech <-chan error + if r.observability != nil { + oech = r.observability.Start(ctx) + } + sech := r.server.ListenAndServe(ctx) + nech, err := r.corrector.StartClient(ctx) + if err != nil { + close(ech) + return nil, err + } + + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-oech: + case err = <-nech: + case err = <-sech: + } + if err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + case ech <- err: + } + } + } + })) + + // main groutine to run the job + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer func() { + log.Info("fiding my pid to kill myself") + p, err := os.FindProcess(os.Getpid()) + if err != nil { + // using Fatal to avoid this process to be zombie + // skipcq: RVV-A0003 + log.Fatalf("failed to find my pid to kill %v", err) + return + } + + log.Info("sending SIGTERM to myself to stop this job") + if err := p.Signal(syscall.SIGTERM); err != nil { + log.Error(err) + } + }() + + start := time.Now() + err = r.corrector.Start(ctx) + if err != nil { + log.Errorf("index correction process failed: %v", err) + return err + } + end := time.Since(start) + log.Infof("correction finished in %v", end) + return nil + })) + return ech, nil +} + +func (r *run) PreStop(ctx context.Context) error { + return r.corrector.PreStop(ctx) +} + +func (r *run) Stop(ctx context.Context) (errs error) { + if r.observability != nil { + if err := r.observability.Stop(ctx); err != nil { + errs = errors.Join(errs, err) + } + } + if r.server != nil { + if err := r.server.Shutdown(ctx); err != nil { + errs = errors.Join(errs, err) + } + } + return errs +} + +func (*run) PostStop(_ context.Context) error { + return nil +} diff --git a/pkg/index/job/correction/usecase/corrector_test.go b/pkg/index/job/correction/usecase/corrector_test.go new file mode 100644 index 0000000000..c8759d29e4 --- /dev/null +++ b/pkg/index/job/correction/usecase/corrector_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// cfg *config.Data +// } +// type want struct { +// wantR runner.Runner +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, runner.Runner, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotR runner.Runner, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotR, w.wantR) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotR, w.wantR) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotR, err := New(test.args.cfg) +// if err := checkFunc(test.want, gotR, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/creation/config/config.go b/pkg/index/job/creation/config/config.go new file mode 100644 index 0000000000..7755c1b4bc --- /dev/null +++ b/pkg/index/job/creation/config/config.go @@ -0,0 +1,71 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" +) + +// GlobalConfig is a type alias of config.GlobalConfig representing application base configurations. +type GlobalConfig = config.GlobalConfig + +// Data represents the application configurations. +type Data struct { + // GlobalConfig represents application base configurations. + config.GlobalConfig `json:",inline" yaml:",inline"` + + // Server represent all server configurations + Server *config.Servers `json:"server_config" yaml:"server_config"` + + // Observability represents observability configurations. + Observability *config.Observability `json:"observability" yaml:"observability"` + + // Creation represents auto indexing service configurations. + Creation *config.IndexCreation `json:"creator" yaml:"creator"` +} + +// NewConfig load configurations from file path. +func NewConfig(path string) (cfg *Data, err error) { + cfg = new(Data) + + if err = config.Read(path, &cfg); err != nil { + return nil, err + } + + if cfg != nil { + _ = cfg.GlobalConfig.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Server != nil { + _ = cfg.Server.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Observability != nil { + _ = cfg.Observability.Bind() + } else { + cfg.Observability = new(config.Observability).Bind() + } + + if cfg.Creation != nil { + _ = cfg.Creation.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + return cfg, nil +} diff --git a/pkg/index/job/creation/config/config_test.go b/pkg/index/job/creation/config/config_test.go new file mode 100644 index 0000000000..0cf6858bf7 --- /dev/null +++ b/pkg/index/job/creation/config/config_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestNewConfig(t *testing.T) { +// type args struct { +// path string +// } +// type want struct { +// wantCfg *Data +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, *Data, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCfg *Data, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCfg, w.wantCfg) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCfg, w.wantCfg) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotCfg, err := NewConfig(test.args.path) +// if err := checkFunc(test.want, gotCfg, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/creation/service/indexer.go b/pkg/index/job/creation/service/indexer.go new file mode 100644 index 0000000000..bd0ff64ba3 --- /dev/null +++ b/pkg/index/job/creation/service/indexer.go @@ -0,0 +1,222 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "context" + "reflect" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" +) + +const ( + apiName = "vald/index/job/create" + grpcMethodName = "core.v1.Agent/" + agent.CreateIndexRPCName +) + +// Indexer represents an interface for indexing. +type Indexer interface { + StartClient(ctx context.Context) (<-chan error, error) + Start(ctx context.Context) error +} + +type index struct { + client discoverer.Client + targetAddrs []string + targetAddrList map[string]bool + + creationPoolSize uint32 + concurrency int +} + +// New returns Indexer object if no error occurs. +func New(opts ...Option) (Indexer, error) { + idx := new(index) + for _, opt := range append(defaultOpts, opts...) { + if err := opt(idx); err != nil { + oerr := errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + e := &errors.ErrCriticalOption{} + if errors.As(oerr, &e) { + log.Error(err) + return nil, oerr + } + log.Warn(oerr) + } + } + idx.targetAddrList = make(map[string]bool, len(idx.targetAddrs)) + for _, addr := range idx.targetAddrs { + idx.targetAddrList[addr] = true + } + return idx, nil +} + +// StartClient starts the gRPC client. +func (idx *index) StartClient(ctx context.Context) (<-chan error, error) { + return idx.client.Start(ctx) +} + +// Start starts indexing process. +func (idx *index) Start(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, apiName+"/service/index.Start") + defer func() { + if span != nil { + span.End() + } + }() + + err := idx.doCreateIndex(ctx, + func(ctx context.Context, ac agent.AgentClient, copts ...grpc.CallOption) (*payload.Empty, error) { + return ac.CreateIndex(ctx, &payload.Control_CreateIndexRequest{ + PoolSize: idx.creationPoolSize, + }, copts...) + }, + ) + if err != nil { + var attrs trace.Attributes + switch { + case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): + err = status.WrapWithInternal( + agent.CreateIndexRPCName+" API connection not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + case errors.Is(err, errors.ErrGRPCTargetAddrNotFound): + err = status.WrapWithInternal( + agent.CreateIndexRPCName+" API connection target address \""+strings.Join(idx.targetAddrs, ",")+"\" not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + default: + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Internal, + "failed to parse "+agent.CreateIndexRPCName+" gRPC error response", + ) + attrs = trace.FromGRPCStatus(st.Code(), msg) + } + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return err + } + return nil +} + +func (idx *index) doCreateIndex(ctx context.Context, fn func(_ context.Context, _ agent.AgentClient, _ ...grpc.CallOption) (*payload.Empty, error)) (errs error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, grpcMethodName), apiName+"/service/index.doCreateIndex") + defer func() { + if span != nil { + span.End() + } + }() + + targetAddrs := idx.client.GetAddrs(ctx) + if len(idx.targetAddrs) != 0 { + targetAddrs = idx.extractTargetAddrs(targetAddrs) + + // If targetAddrs is empty, an invalid target addresses may be registered in targetAddrList. + if len(targetAddrs) == 0 { + return errors.ErrGRPCTargetAddrNotFound + } + } + log.Infof("target agent addrs: %v", targetAddrs) + + var emu sync.Mutex + err := idx.client.GetClient().OrderedRangeConcurrent(ctx, targetAddrs, idx.concurrency, + func(ctx context.Context, target string, conn *grpc.ClientConn, copts ...grpc.CallOption) error { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "OrderedRangeConcurrent/"+target), agent.CreateIndexRPCName+"/"+target) + defer func() { + if span != nil { + span.End() + } + }() + _, err := fn(ctx, agent.NewAgentClient(conn), copts...) + if err != nil { + var attrs trace.Attributes + switch { + case errors.Is(err, context.Canceled): + err = status.WrapWithCanceled( + agent.CreateIndexRPCName+" API canceld", err, + ) + attrs = trace.StatusCodeCancelled(err.Error()) + case errors.Is(err, context.DeadlineExceeded): + err = status.WrapWithCanceled( + agent.CreateIndexRPCName+" API deadline exceeded", err, + ) + attrs = trace.StatusCodeDeadlineExceeded(err.Error()) + case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): + err = status.WrapWithInternal( + agent.CreateIndexRPCName+" API connection not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + case errors.Is(err, errors.ErrTargetNotFound): + err = status.WrapWithInvalidArgument( + agent.CreateIndexRPCName+" API target not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + default: + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Internal, + "failed to parse "+agent.CreateIndexRPCName+" gRPC error response", + ) + if st != nil && err != nil && st.Code() == codes.FailedPrecondition { + log.Warnf("CreateIndex of %s skipped, message: %s, err: %v", target, st.Message(), errors.Join(st.Err(), err)) + return nil + } + attrs = trace.FromGRPCStatus(st.Code(), msg) + } + log.Warnf("an error occurred in (%s) during indexing: %v", target, err) + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + emu.Lock() + errs = errors.Join(errs, err) + emu.Unlock() + } + return err + }, + ) + return errors.Join(err, errs) +} + +// extractTargetAddresses filters and extracts target addresses registered in targetAddrList from the given address list. +func (idx *index) extractTargetAddrs(addrs []string) []string { + res := make([]string, 0, len(addrs)) + for _, addr := range addrs { + if !idx.targetAddrList[addr] { + log.Warnf("the gRPC target address not found: %s", addr) + } else { + res = append(res, addr) + } + } + return res +} diff --git a/pkg/index/job/creation/service/indexer_test.go b/pkg/index/job/creation/service/indexer_test.go new file mode 100644 index 0000000000..6229dd21ee --- /dev/null +++ b/pkg/index/job/creation/service/indexer_test.go @@ -0,0 +1,425 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "context" + "testing" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/test/goleak" + clientmock "github.com/vdaas/vald/internal/test/mock/client" + grpcmock "github.com/vdaas/vald/internal/test/mock/grpc" +) + +func Test_index_Start(t *testing.T) { + t.Parallel() + type args struct { + ctx context.Context + } + type fields struct { + client discoverer.Client + targetAddrs []string + targetAddrList map[string]bool + creationPoolSize uint32 + concurrency int + } + type want struct { + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, error) error + beforeFunc func(*testing.T, args) + afterFunc func(*testing.T, args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + return nil + } + tests := []test{ + func() test { + addrs := []string{ + "127.0.0.1:8080", + } + return test{ + name: "Success: when there is no error in the indexing request process", + args: args{ + ctx: context.Background(), + }, + + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + return addrs + }, + GetClientFunc: func() grpc.Client { + return &grpcmock.GRPCClientMock{ + OrderedRangeConcurrentFunc: func(_ context.Context, _ []string, _ int, + _ func(_ context.Context, _ string, _ *grpc.ClientConn, _ ...grpc.CallOption) error, + ) error { + return nil + }, + } + }, + }, + }, + } + }(), + func() test { + addrs := []string{ + "127.0.0.1:8080", + } + return test{ + name: "Fail: when there is an error wrapped with gRPC status in the indexing request process", + args: args{ + ctx: context.Background(), + }, + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + return addrs + }, + GetClientFunc: func() grpc.Client { + return &grpcmock.GRPCClientMock{ + OrderedRangeConcurrentFunc: func(_ context.Context, _ []string, _ int, + _ func(_ context.Context, _ string, _ *grpc.ClientConn, _ ...grpc.CallOption) error, + ) error { + return status.WrapWithInternal( + agent.CreateIndexRPCName+" API connection not found", + errors.ErrGRPCClientConnNotFound("*"), + ) + }, + } + }, + }, + }, + want: want{ + err: status.Error(codes.Internal, + agent.CreateIndexRPCName+" API connection not found"), + }, + } + }(), + func() test { + addrs := []string{ + "127.0.0.1:8080", + } + return test{ + name: "Fail: When the OrderedRangeConcurrent method returns a gRPC client conn not found error", + args: args{ + ctx: context.Background(), + }, + + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + return addrs + }, + GetClientFunc: func() grpc.Client { + return &grpcmock.GRPCClientMock{ + OrderedRangeConcurrentFunc: func(_ context.Context, _ []string, _ int, + _ func(_ context.Context, _ string, _ *grpc.ClientConn, _ ...grpc.CallOption) error, + ) error { + return errors.ErrGRPCClientConnNotFound("*") + }, + } + }, + }, + }, + want: want{ + err: status.Error(codes.Internal, + agent.CreateIndexRPCName+" API connection not found"), + }, + } + }(), + func() test { + targetAddrs := []string{ + "127.0.0.1:8080", + } + targetAddrList := map[string]bool{ + targetAddrs[0]: true, + } + return test{ + name: "Fail: when there is no address matching targetAddrList", + args: args{ + ctx: context.Background(), + }, + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + return nil + }, + }, + targetAddrs: targetAddrs, + targetAddrList: targetAddrList, + }, + want: want{ + err: status.Error(codes.Internal, + agent.CreateIndexRPCName+" API connection target address \"127.0.0.1:8080\" not found"), + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(tt, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + idx := &index{ + client: test.fields.client, + targetAddrs: test.fields.targetAddrs, + targetAddrList: test.fields.targetAddrList, + creationPoolSize: test.fields.creationPoolSize, + concurrency: test.fields.concurrency, + } + + err := idx.Start(test.args.ctx) + if err := checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// opts []Option +// } +// type want struct { +// want Indexer +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Indexer, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Indexer, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, err := New(test.args.opts...) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_index_StartClient(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// client discoverer.Client +// targetAddrs []string +// targetAddrList map[string]bool +// creationPoolSize uint32 +// concurrency int +// } +// type want struct { +// want <-chan error +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, <-chan error, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got <-chan error, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// client:nil, +// targetAddrs:nil, +// targetAddrList:nil, +// creationPoolSize:0, +// concurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// client:nil, +// targetAddrs:nil, +// targetAddrList:nil, +// creationPoolSize:0, +// concurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// idx := &index{ +// client: test.fields.client, +// targetAddrs: test.fields.targetAddrs, +// targetAddrList: test.fields.targetAddrList, +// creationPoolSize: test.fields.creationPoolSize, +// concurrency: test.fields.concurrency, +// } +// +// got, err := idx.StartClient(test.args.ctx) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/creation/service/options.go b/pkg/index/job/creation/service/options.go new file mode 100644 index 0000000000..b6c7322a25 --- /dev/null +++ b/pkg/index/job/creation/service/options.go @@ -0,0 +1,70 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/errors" +) + +// Option represents the functional option for index. +type Option func(_ *index) error + +var defaultOpts = []Option{ + WithIndexingConcurrency(1), + WithCreationPoolSize(1000), +} + +// WithDiscoverer returns Option that sets discoverer client. +func WithDiscoverer(client discoverer.Client) Option { + return func(idx *index) error { + if client == nil { + return errors.NewErrCriticalOption("discoverer", client) + } + idx.client = client + return nil + } +} + +// WithIndexingConcurrency returns Option that sets indexing concurrency. +func WithIndexingConcurrency(num int) Option { + return func(idx *index) error { + if num <= 0 { + return errors.NewErrInvalidOption("indexingConcurrency", num) + } + idx.concurrency = num + return nil + } +} + +// WithCreationPoolSize returns Option that sets indexing pool size. +func WithCreationPoolSize(size uint32) Option { + return func(idx *index) error { + if size <= 0 { + return errors.NewErrInvalidOption("creationPoolSize", size) + } + idx.creationPoolSize = size + return nil + } +} + +// WithTargetAddrs returns Option that sets indexing target addresses. +func WithTargetAddrs(addrs ...string) Option { + return func(idx *index) error { + if len(addrs) != 0 { + idx.targetAddrs = append(idx.targetAddrs, addrs...) + } + return nil + } +} diff --git a/pkg/index/job/creation/service/options_test.go b/pkg/index/job/creation/service/options_test.go new file mode 100644 index 0000000000..56b7b34cd8 --- /dev/null +++ b/pkg/index/job/creation/service/options_test.go @@ -0,0 +1,360 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +// NOT IMPLEMENTED BELOW +// +// func TestWithDiscoverer(t *testing.T) { +// type args struct { +// client discoverer.Client +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// client:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// client:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithDiscoverer(test.args.client) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithIndexingConcurrency(t *testing.T) { +// type args struct { +// num int +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithIndexingConcurrency(test.args.num) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithCreationPoolSize(t *testing.T) { +// type args struct { +// size uint32 +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// size:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// size:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithCreationPoolSize(test.args.size) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithTargetAddrs(t *testing.T) { +// type args struct { +// addrs []string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// addrs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// addrs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithTargetAddrs(test.args.addrs...) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/creation/usecase/creation.go b/pkg/index/job/creation/usecase/creation.go new file mode 100644 index 0000000000..932263c7e8 --- /dev/null +++ b/pkg/index/job/creation/usecase/creation.go @@ -0,0 +1,214 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +import ( + "context" + "os" + "syscall" + + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + iconfig "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/interceptor/server/recover" + "github.com/vdaas/vald/internal/observability" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" + "github.com/vdaas/vald/pkg/index/job/creation/config" + "github.com/vdaas/vald/pkg/index/job/creation/service" +) + +type run struct { + eg errgroup.Group + cfg *config.Data + observability observability.Observability + server starter.Server + indexer service.Indexer +} + +// New returns Runner instance. +func New(cfg *config.Data) (_ runner.Runner, err error) { + eg := errgroup.Get() + + dOpts, err := cfg.Creation.Discoverer.Client.Opts() + if err != nil { + return nil, err + } + // skipcq: CRT-D0001 + dOpts = append(dOpts, grpc.WithErrGroup(eg)) + + acOpts, err := cfg.Creation.Discoverer.AgentClientOptions.Opts() + if err != nil { + return nil, err + } + // skipcq: CRT-D0001 + acOpts = append(acOpts, grpc.WithErrGroup(eg)) + + discoverer, err := discoverer.New( + discoverer.WithAutoConnect(true), + discoverer.WithName(cfg.Creation.AgentName), + discoverer.WithNamespace(cfg.Creation.AgentNamespace), + discoverer.WithPort(cfg.Creation.AgentPort), + discoverer.WithServiceDNSARecord(cfg.Creation.AgentDNS), + discoverer.WithDiscovererClient(grpc.New(dOpts...)), + discoverer.WithDiscoverDuration(cfg.Creation.Discoverer.Duration), + discoverer.WithOptions(acOpts...), + discoverer.WithNodeName(cfg.Creation.NodeName), + discoverer.WithOnDiscoverFunc(func(ctx context.Context, c discoverer.Client, addrs []string) error { + last := len(addrs) - 1 + for i := 0; i < len(addrs)/2; i++ { + addrs[i], addrs[last-i] = addrs[last-i], addrs[i] + } + return nil + }), + ) + if err != nil { + return nil, err + } + + indexer, err := service.New( + service.WithDiscoverer(discoverer), + service.WithIndexingConcurrency(cfg.Creation.Concurrency), + service.WithCreationPoolSize(cfg.Creation.CreationPoolSize), + service.WithTargetAddrs(cfg.Creation.TargetAddrs...), + ) + if err != nil { + return nil, err + } + + srv, err := starter.New( + starter.WithConfig(cfg.Server), + starter.WithGRPC(func(cfg *iconfig.Server) []server.Option { + return []server.Option{ + server.WithGRPCOption( + grpc.ChainUnaryInterceptor(recover.RecoverInterceptor()), + grpc.ChainStreamInterceptor(recover.RecoverStreamInterceptor()), + ), + } + }), + ) + if err != nil { + return nil, err + } + + var obs observability.Observability + if cfg.Observability.Enabled { + obs, err = observability.NewWithConfig( + cfg.Observability, + ) + if err != nil { + return nil, err + } + } + + return &run{ + eg: eg, + cfg: cfg, + observability: obs, + server: srv, + indexer: indexer, + }, nil +} + +// PreStart is a method called before execution of Start, and it invokes the PreStart method of observability. +func (r *run) PreStart(ctx context.Context) error { + if r.observability != nil { + return r.observability.PreStart(ctx) + } + return nil +} + +// Start is a method used to initiate an operation in the run, and it returns a channel for receiving errors +// during the operation and an error representing any initialization errors. +func (r *run) Start(ctx context.Context) (<-chan error, error) { + ech := make(chan error, 4) + var sech, oech <-chan error + if r.observability != nil { + oech = r.observability.Start(ctx) + } + sech = r.server.ListenAndServe(ctx) + cech, err := r.indexer.StartClient(ctx) + if err != nil { + close(ech) + return nil, err + } + + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer func() { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + // using Fatal to avoid this process to be zombie + // skipcq: RVV-A0003 + log.Fatalf("failed to find my pid to kill %v", err) + return + } + log.Info("sending SIGTERM to myself to stop this job") + if err := p.Signal(syscall.SIGTERM); err != nil { + log.Error(err) + } + }() + return r.indexer.Start(ctx) + })) + + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-oech: + case err = <-sech: + case err = <-cech: + } + if err != nil { + select { + case <-ctx.Done(): + return errors.Join(ctx.Err(), err) + case ech <- err: + } + } + } + })) + return ech, nil +} + +// PreStop is a method called before execution of Stop. +func (*run) PreStop(_ context.Context) error { + return nil +} + +// Stop is a method used to stop an operation in the run. +func (r *run) Stop(ctx context.Context) (errs error) { + if r.observability != nil { + if err := r.observability.Stop(ctx); err != nil { + errs = errors.Join(errs, err) + } + } + if r.server != nil { + if err := r.server.Shutdown(ctx); err != nil { + errs = errors.Join(errs, err) + } + } + return errs +} + +// PtopStop is a method called after execution of Stop. +func (*run) PostStop(_ context.Context) error { + return nil +} diff --git a/pkg/index/job/creation/usecase/creation_test.go b/pkg/index/job/creation/usecase/creation_test.go new file mode 100644 index 0000000000..1eab852c2d --- /dev/null +++ b/pkg/index/job/creation/usecase/creation_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// cfg *config.Data +// } +// type want struct { +// want runner.Runner +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, runner.Runner, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got runner.Runner, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, err := New(test.args.cfg) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/save/config/config.go b/pkg/index/job/save/config/config.go new file mode 100644 index 0000000000..975b871c9c --- /dev/null +++ b/pkg/index/job/save/config/config.go @@ -0,0 +1,71 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" +) + +// GlobalConfig is a type alias of config.GlobalConfig representing application base configurations. +type GlobalConfig = config.GlobalConfig + +// Data represents the application configurations. +type Data struct { + // GlobalConfig represents application base configurations. + config.GlobalConfig `json:",inline" yaml:",inline"` + + // Server represent all server configurations + Server *config.Servers `json:"server_config" yaml:"server_config"` + + // Observability represents observability configurations. + Observability *config.Observability `json:"observability" yaml:"observability"` + + // Save represents auto save indexing service configurations. + Save *config.IndexSave `json:"saver" yaml:"saver"` +} + +// NewConfig loads configurations from the file path. +func NewConfig(path string) (cfg *Data, err error) { + cfg = new(Data) + + if err = config.Read(path, &cfg); err != nil { + return nil, err + } + + if cfg != nil { + _ = cfg.GlobalConfig.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Server != nil { + _ = cfg.Server.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Observability != nil { + _ = cfg.Observability.Bind() + } else { + cfg.Observability = new(config.Observability).Bind() + } + + if cfg.Save != nil { + _ = cfg.Save.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + return cfg, nil +} diff --git a/pkg/index/job/save/config/config_test.go b/pkg/index/job/save/config/config_test.go new file mode 100644 index 0000000000..0cf6858bf7 --- /dev/null +++ b/pkg/index/job/save/config/config_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// NOT IMPLEMENTED BELOW +// +// func TestNewConfig(t *testing.T) { +// type args struct { +// path string +// } +// type want struct { +// wantCfg *Data +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, *Data, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotCfg *Data, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotCfg, w.wantCfg) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotCfg, w.wantCfg) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// path:"", +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// gotCfg, err := NewConfig(test.args.path) +// if err := checkFunc(test.want, gotCfg, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/save/service/indexer.go b/pkg/index/job/save/service/indexer.go new file mode 100644 index 0000000000..b437756061 --- /dev/null +++ b/pkg/index/job/save/service/indexer.go @@ -0,0 +1,215 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "context" + "reflect" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync" +) + +const ( + apiName = "vald/index/job/save" + grpcMethodName = "core.v1.Agent/" + agent.SaveIndexRPCName +) + +// Indexer represents an interface for indexing. +type Indexer interface { + StartClient(ctx context.Context) (<-chan error, error) + Start(ctx context.Context) error +} + +type index struct { + client discoverer.Client + targetAddrs []string + targetAddrList map[string]bool + + concurrency int +} + +// New returns Indexer object if no error occurs. +func New(opts ...Option) (Indexer, error) { + idx := new(index) + for _, opt := range append(defaultOpts, opts...) { + if err := opt(idx); err != nil { + oerr := errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + e := &errors.ErrCriticalOption{} + if errors.As(oerr, &e) { + log.Error(err) + return nil, oerr + } + log.Warn(oerr) + } + } + idx.targetAddrList = make(map[string]bool, len(idx.targetAddrs)) + for _, addr := range idx.targetAddrs { + idx.targetAddrList[addr] = true + } + return idx, nil +} + +// StartClient starts the gRPC client. +func (idx *index) StartClient(ctx context.Context) (<-chan error, error) { + return idx.client.Start(ctx) +} + +// Start starts indexing process. +func (idx *index) Start(ctx context.Context) error { + ctx, span := trace.StartSpan(ctx, apiName+"/service/index.Start") + defer func() { + if span != nil { + span.End() + } + }() + + err := idx.doSaveIndex(ctx, + func(ctx context.Context, ac agent.AgentClient, copts ...grpc.CallOption) (*payload.Empty, error) { + return ac.SaveIndex(ctx, &payload.Empty{}, copts...) + }, + ) + if err != nil { + var attrs trace.Attributes + switch { + case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): + err = status.WrapWithInternal( + agent.SaveIndexRPCName+" API connection not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + case errors.Is(err, errors.ErrGRPCTargetAddrNotFound): + err = status.WrapWithInternal( + agent.SaveIndexRPCName+" API connection target address \""+strings.Join(idx.targetAddrs, ",")+"\" not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + default: + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Internal, + "failed to parse "+agent.SaveIndexRPCName+" gRPC error response", + ) + attrs = trace.FromGRPCStatus(st.Code(), msg) + } + log.Warn(err) + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + return err + } + return nil +} + +func (idx *index) doSaveIndex(ctx context.Context, fn func(_ context.Context, _ agent.AgentClient, _ ...grpc.CallOption) (*payload.Empty, error)) (errs error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, grpcMethodName), apiName+"/service/index.doSaveIndex") + defer func() { + if span != nil { + span.End() + } + }() + + targetAddrs := idx.client.GetAddrs(ctx) + if len(idx.targetAddrs) != 0 { + targetAddrs = idx.extractTargetAddrs(targetAddrs) + + // If targetAddrs is empty, an invalid target addresses may be registered in targetAddrList. + if len(targetAddrs) == 0 { + return errors.ErrGRPCTargetAddrNotFound + } + } + log.Infof("target agent addrs: %v", targetAddrs) + + var emu sync.Mutex + err := idx.client.GetClient().OrderedRangeConcurrent(ctx, targetAddrs, idx.concurrency, + func(ctx context.Context, target string, conn *grpc.ClientConn, copts ...grpc.CallOption) error { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "OrderedRangeConcurrent/"+target), agent.SaveIndexRPCName+"/"+target) + defer func() { + if span != nil { + span.End() + } + }() + _, err := fn(ctx, agent.NewAgentClient(conn), copts...) + if err != nil { + var attrs trace.Attributes + switch { + case errors.Is(err, context.Canceled): + err = status.WrapWithCanceled( + agent.SaveIndexRPCName+" API canceld", err, + ) + attrs = trace.StatusCodeCancelled(err.Error()) + case errors.Is(err, context.DeadlineExceeded): + err = status.WrapWithCanceled( + agent.SaveIndexRPCName+" API deadline exceeded", err, + ) + attrs = trace.StatusCodeDeadlineExceeded(err.Error()) + case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): + err = status.WrapWithInternal( + agent.SaveIndexRPCName+" API connection not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + case errors.Is(err, errors.ErrTargetNotFound): + err = status.WrapWithInvalidArgument( + agent.SaveIndexRPCName+" API target not found", err, + ) + attrs = trace.StatusCodeInternal(err.Error()) + default: + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Internal, + "failed to parse "+agent.SaveIndexRPCName+" gRPC error response", + ) + attrs = trace.FromGRPCStatus(st.Code(), msg) + } + log.Warnf("an error occurred in (%s) during save indexing: %v", target, err) + if span != nil { + span.RecordError(err) + span.SetAttributes(attrs...) + span.SetStatus(trace.StatusError, err.Error()) + } + emu.Lock() + errs = errors.Join(errs, err) + emu.Unlock() + } + return err + }, + ) + return errors.Join(err, errs) +} + +// extractTargetAddresses filters and extracts target addresses registered in targetAddrList from the given address list. +func (idx *index) extractTargetAddrs(addrs []string) []string { + res := make([]string, 0, len(addrs)) + for _, addr := range addrs { + if !idx.targetAddrList[addr] { + log.Warnf("the gRPC target address not found: %s", addr) + } else { + res = append(res, addr) + } + } + return res +} diff --git a/pkg/index/job/save/service/indexer_test.go b/pkg/index/job/save/service/indexer_test.go new file mode 100644 index 0000000000..90d97b2c09 --- /dev/null +++ b/pkg/index/job/save/service/indexer_test.go @@ -0,0 +1,419 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "context" + "testing" + + agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/codes" + "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/test/goleak" + clientmock "github.com/vdaas/vald/internal/test/mock/client" + grpcmock "github.com/vdaas/vald/internal/test/mock/grpc" +) + +func Test_index_Start(t *testing.T) { + type args struct { + ctx context.Context + } + type fields struct { + client discoverer.Client + targetAddrs []string + targetAddrList map[string]bool + concurrency int + } + type want struct { + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, error) error + beforeFunc func(*testing.T, args) + afterFunc func(*testing.T, args) + } + defaultCheckFunc := func(w want, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) + } + return nil + } + tests := []test{ + func() test { + addrs := []string{ + "127.0.0.1:8080", + } + return test{ + name: "Success: when there is no error in the save indexing request process", + args: args{ + ctx: context.Background(), + }, + + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + return addrs + }, + GetClientFunc: func() grpc.Client { + return &grpcmock.GRPCClientMock{ + OrderedRangeConcurrentFunc: func(_ context.Context, _ []string, _ int, + _ func(_ context.Context, _ string, _ *grpc.ClientConn, _ ...grpc.CallOption) error, + ) error { + return nil + }, + } + }, + }, + }, + } + }(), + func() test { + addrs := []string{ + "127.0.0.1:8080", + } + return test{ + name: "Fail: when there is an error wrapped with gRPC status in the save indexing request process", + args: args{ + ctx: context.Background(), + }, + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + return addrs + }, + GetClientFunc: func() grpc.Client { + return &grpcmock.GRPCClientMock{ + OrderedRangeConcurrentFunc: func(_ context.Context, _ []string, _ int, + _ func(_ context.Context, _ string, _ *grpc.ClientConn, _ ...grpc.CallOption) error, + ) error { + return status.WrapWithInternal( + agent.SaveIndexRPCName+" API connection not found", + errors.ErrGRPCClientConnNotFound("*"), + ) + }, + } + }, + }, + }, + want: want{ + err: status.Error(codes.Internal, + agent.SaveIndexRPCName+" API connection not found"), + }, + } + }(), + func() test { + addrs := []string{ + "127.0.0.1:8080", + } + return test{ + name: "Fail: When the OrderedRangeConcurrent method returns a gRPC client conn not found error", + args: args{ + ctx: context.Background(), + }, + + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + return addrs + }, + GetClientFunc: func() grpc.Client { + return &grpcmock.GRPCClientMock{ + OrderedRangeConcurrentFunc: func(_ context.Context, _ []string, _ int, + _ func(_ context.Context, _ string, _ *grpc.ClientConn, _ ...grpc.CallOption) error, + ) error { + return errors.ErrGRPCClientConnNotFound("*") + }, + } + }, + }, + }, + want: want{ + err: status.Error(codes.Internal, + agent.SaveIndexRPCName+" API connection not found"), + }, + } + }(), + func() test { + targetAddrs := []string{ + "127.0.0.1:8080", + } + targetAddrList := map[string]bool{ + targetAddrs[0]: true, + } + return test{ + name: "Fail: when there is no address matching targetAddrList", + args: args{ + ctx: context.Background(), + }, + fields: fields{ + client: &clientmock.DiscovererClientMock{ + GetAddrsFunc: func(_ context.Context) []string { + // NOTE: This function returns nil, meaning that the targetAddrs stored in the field are invalid values. + return nil + }, + }, + targetAddrs: targetAddrs, + targetAddrList: targetAddrList, + }, + want: want{ + err: status.Error(codes.Internal, + agent.SaveIndexRPCName+" API connection target address \"127.0.0.1:8080\" not found"), + }, + } + }(), + } + + for _, tc := range tests { + test := tc + t.Run(test.name, func(tt *testing.T) { + tt.Parallel() + defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) + if test.beforeFunc != nil { + test.beforeFunc(tt, test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(tt, test.args) + } + checkFunc := test.checkFunc + if test.checkFunc == nil { + checkFunc = defaultCheckFunc + } + idx := &index{ + client: test.fields.client, + targetAddrs: test.fields.targetAddrs, + targetAddrList: test.fields.targetAddrList, + concurrency: test.fields.concurrency, + } + + err := idx.Start(test.args.ctx) + if err := checkFunc(test.want, err); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// opts []Option +// } +// type want struct { +// want Indexer +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Indexer, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Indexer, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// opts:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, err := New(test.args.opts...) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func Test_index_StartClient(t *testing.T) { +// type args struct { +// ctx context.Context +// } +// type fields struct { +// client discoverer.Client +// targetAddrs []string +// targetAddrList map[string]bool +// concurrency int +// } +// type want struct { +// want <-chan error +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, <-chan error, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got <-chan error, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// client:nil, +// targetAddrs:nil, +// targetAddrList:nil, +// concurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// }, +// fields: fields { +// client:nil, +// targetAddrs:nil, +// targetAddrList:nil, +// concurrency:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// idx := &index{ +// client: test.fields.client, +// targetAddrs: test.fields.targetAddrs, +// targetAddrList: test.fields.targetAddrList, +// concurrency: test.fields.concurrency, +// } +// +// got, err := idx.StartClient(test.args.ctx) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/save/service/options.go b/pkg/index/job/save/service/options.go new file mode 100644 index 0000000000..bd4c997e37 --- /dev/null +++ b/pkg/index/job/save/service/options.go @@ -0,0 +1,58 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + "github.com/vdaas/vald/internal/errors" +) + +// Option represents the functional option for index. +type Option func(_ *index) error + +var defaultOpts = []Option{ + WithSaveIndexingConcurrency(1), +} + +// WithDiscoverer returns Option that sets discoverer client. +func WithDiscoverer(client discoverer.Client) Option { + return func(idx *index) error { + if client == nil { + return errors.NewErrCriticalOption("discoverer", client) + } + idx.client = client + return nil + } +} + +// WithSaveIndexingConcurrency returns Option that sets save indexing concurrency. +func WithSaveIndexingConcurrency(num int) Option { + return func(idx *index) error { + if num <= 0 { + return errors.NewErrInvalidOption("saveIndexingConcurrency", num) + } + idx.concurrency = num + return nil + } +} + +// WithTargetAddrs returns Option that sets indexing target addresses. +func WithTargetAddrs(addrs ...string) Option { + return func(idx *index) error { + if len(addrs) != 0 { + idx.targetAddrs = append(idx.targetAddrs, addrs...) + } + return nil + } +} diff --git a/pkg/index/job/save/service/options_test.go b/pkg/index/job/save/service/options_test.go new file mode 100644 index 0000000000..b8137a55e6 --- /dev/null +++ b/pkg/index/job/save/service/options_test.go @@ -0,0 +1,274 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +// NOT IMPLEMENTED BELOW +// +// func TestWithDiscoverer(t *testing.T) { +// type args struct { +// client discoverer.Client +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// client:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// client:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithDiscoverer(test.args.client) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithSaveIndexingConcurrency(t *testing.T) { +// type args struct { +// num int +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// num:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithSaveIndexingConcurrency(test.args.num) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } +// +// func TestWithTargetAddrs(t *testing.T) { +// type args struct { +// addrs []string +// } +// type want struct { +// want Option +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, Option) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got Option) error { +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// addrs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// addrs:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got := WithTargetAddrs(test.args.addrs...) +// if err := checkFunc(test.want, got); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/index/job/save/usecase/save.go b/pkg/index/job/save/usecase/save.go new file mode 100644 index 0000000000..38327e2f1d --- /dev/null +++ b/pkg/index/job/save/usecase/save.go @@ -0,0 +1,213 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +import ( + "context" + "os" + "syscall" + + "github.com/vdaas/vald/internal/client/v1/client/discoverer" + iconfig "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/interceptor/server/recover" + "github.com/vdaas/vald/internal/observability" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/servers/server" + "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" + "github.com/vdaas/vald/pkg/index/job/save/config" + "github.com/vdaas/vald/pkg/index/job/save/service" +) + +type run struct { + eg errgroup.Group + cfg *config.Data + observability observability.Observability + server starter.Server + indexer service.Indexer +} + +// New returns Runner instance. +func New(cfg *config.Data) (_ runner.Runner, err error) { + eg := errgroup.Get() + + dOpts, err := cfg.Save.Discoverer.Client.Opts() + if err != nil { + return nil, err + } + // skipcq: CRT-D0001 + dOpts = append(dOpts, grpc.WithErrGroup(eg)) + + acOpts, err := cfg.Save.Discoverer.AgentClientOptions.Opts() + if err != nil { + return nil, err + } + // skipcq: CRT-D0001 + acOpts = append(acOpts, grpc.WithErrGroup(eg)) + + discoverer, err := discoverer.New( + discoverer.WithAutoConnect(true), + discoverer.WithName(cfg.Save.AgentName), + discoverer.WithNamespace(cfg.Save.AgentNamespace), + discoverer.WithPort(cfg.Save.AgentPort), + discoverer.WithServiceDNSARecord(cfg.Save.AgentDNS), + discoverer.WithDiscovererClient(grpc.New(dOpts...)), + discoverer.WithDiscoverDuration(cfg.Save.Discoverer.Duration), + discoverer.WithOptions(acOpts...), + discoverer.WithNodeName(cfg.Save.NodeName), + discoverer.WithOnDiscoverFunc(func(ctx context.Context, c discoverer.Client, addrs []string) error { + last := len(addrs) - 1 + for i := 0; i < len(addrs)/2; i++ { + addrs[i], addrs[last-i] = addrs[last-i], addrs[i] + } + return nil + }), + ) + if err != nil { + return nil, err + } + + indexer, err := service.New( + service.WithDiscoverer(discoverer), + service.WithSaveIndexingConcurrency(cfg.Save.Concurrency), + service.WithTargetAddrs(cfg.Save.TargetAddrs...), + ) + if err != nil { + return nil, err + } + + srv, err := starter.New( + starter.WithConfig(cfg.Server), + starter.WithGRPC(func(cfg *iconfig.Server) []server.Option { + return []server.Option{ + server.WithGRPCOption( + grpc.ChainUnaryInterceptor(recover.RecoverInterceptor()), + grpc.ChainStreamInterceptor(recover.RecoverStreamInterceptor()), + ), + } + }), + ) + if err != nil { + return nil, err + } + + var obs observability.Observability + if cfg.Observability.Enabled { + obs, err = observability.NewWithConfig( + cfg.Observability, + ) + if err != nil { + return nil, err + } + } + + return &run{ + eg: eg, + cfg: cfg, + observability: obs, + server: srv, + indexer: indexer, + }, nil +} + +// PreStart is a method called before execution of Start, and it invokes the PreStart method of observability. +func (r *run) PreStart(ctx context.Context) error { + if r.observability != nil { + return r.observability.PreStart(ctx) + } + return nil +} + +// Start is a method used to initiate an operation in the run, and it returns a channel for receiving errors +// during the operation and an error representing any initialization errors. +func (r *run) Start(ctx context.Context) (<-chan error, error) { + ech := make(chan error, 4) + var sech, oech <-chan error + if r.observability != nil { + oech = r.observability.Start(ctx) + } + sech = r.server.ListenAndServe(ctx) + cech, err := r.indexer.StartClient(ctx) + if err != nil { + close(ech) + return nil, err + } + + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer func() { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + // using Fatal to avoid this process to be zombie + // skipcq: RVV-A0003 + log.Fatalf("failed to find my pid to kill %v", err) + return + } + log.Info("sending SIGTERM to myself to stop this job") + if err := p.Signal(syscall.SIGTERM); err != nil { + log.Error(err) + } + }() + return r.indexer.Start(ctx) + })) + + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-oech: + case err = <-sech: + case err = <-cech: + } + if err != nil { + select { + case <-ctx.Done(): + return errors.Join(ctx.Err(), err) + case ech <- err: + } + } + } + })) + return ech, nil +} + +// PreStop is a method called before execution of Stop. +func (*run) PreStop(_ context.Context) error { + return nil +} + +// Stop is a method used to stop an operation in the run. +func (r *run) Stop(ctx context.Context) (errs error) { + if r.observability != nil { + if err := r.observability.Stop(ctx); err != nil { + errs = errors.Join(errs, err) + } + } + if r.server != nil { + if err := r.server.Shutdown(ctx); err != nil { + errs = errors.Join(errs, err) + } + } + return errs +} + +// PtopStop is a method called after execution of Stop. +func (*run) PostStop(_ context.Context) error { + return nil +} diff --git a/pkg/index/job/save/usecase/save_test.go b/pkg/index/job/save/usecase/save_test.go new file mode 100644 index 0000000000..1eab852c2d --- /dev/null +++ b/pkg/index/job/save/usecase/save_test.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +// NOT IMPLEMENTED BELOW +// +// func TestNew(t *testing.T) { +// type args struct { +// cfg *config.Data +// } +// type want struct { +// want runner.Runner +// err error +// } +// type test struct { +// name string +// args args +// want want +// checkFunc func(want, runner.Runner, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, got runner.Runner, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(got, w.want) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", got, w.want) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// cfg:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// +// got, err := New(test.args.cfg) +// if err := checkFunc(test.want, got, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/tests/chaos/chart/README.md b/tests/chaos/chart/README.md index 0a5dbdc2cb..2879392f10 100644 --- a/tests/chaos/chart/README.md +++ b/tests/chaos/chart/README.md @@ -41,4 +41,4 @@ A Helm chart for testing Vald using Chaos Mesh. --- -Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) +Autogenerated from chart metadata using [helm-docs v1.11.3](https://github.com/norwoodj/helm-docs/releases/v1.11.3) diff --git a/tests/e2e/crud/crud_test.go b/tests/e2e/crud/crud_test.go index 97e3428e08..bb2ce4c2a5 100644 --- a/tests/e2e/crud/crud_test.go +++ b/tests/e2e/crud/crud_test.go @@ -24,6 +24,7 @@ import ( "flag" "fmt" "os" + "os/exec" "testing" "time" @@ -42,13 +43,14 @@ var ( port int ds *hdf5.Dataset - insertNum int - searchNum int - searchByIDNum int - getObjectNum int - updateNum int - upsertNum int - removeNum int + insertNum int + correctionInsertNum int + searchNum int + searchByIDNum int + getObjectNum int + updateNum int + upsertNum int + removeNum int insertFrom int searchFrom int @@ -73,6 +75,7 @@ func init() { flag.IntVar(&port, "port", 8081, "gRPC port") flag.IntVar(&insertNum, "insert-num", 10000, "number of id-vector pairs used for insert") + flag.IntVar(&correctionInsertNum, "correction-insert-num", 3000, "number of id-vector pairs used for insert") flag.IntVar(&searchNum, "search-num", 10000, "number of id-vector pairs used for search") flag.IntVar(&searchByIDNum, "search-by-id-num", 100, "number of id-vector pairs used for search-by-id") flag.IntVar(&getObjectNum, "get-object-num", 100, "number of id-vector pairs used for get-object") @@ -375,6 +378,13 @@ func TestE2EStandardCRUD(t *testing.T) { t.Fatalf("an error occurred: %s", err) } + err = op.StreamListObject(t, ctx, operation.Dataset{ + Train: ds.Train[insertFrom : insertFrom+insertNum], + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + err = op.Update(t, ctx, operation.Dataset{ Train: ds.Train[updateFrom : updateFrom+updateNum], }) @@ -738,3 +748,78 @@ func TestE2ECRUDWithSkipStrictExistCheck(t *testing.T) { t.Fatalf("an error occurred on #13: %s", err) } } + +// TestE2EIndexJobCorrection tests the index correction job. +// It inserts vectors, runs the index correction job, and then removes the vectors. +// TODO: Add index replica count check after inplementing StreamListObject in LB +func TestE2EIndexJobCorrection(t *testing.T) { + t.Cleanup(teardown) + ctx := context.Background() + + op, err := operation.New(host, port) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + // prepare train data + train := ds.Train[insertFrom : insertFrom+correctionInsertNum] + + err = op.Insert(t, ctx, operation.Dataset{ + Train: train, + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + sleep(t, waitAfterInsertDuration) + + t.Log("Test case 1: just execute index correction and check if replica number is correct after correction") + exe := operation.NewCronJobExecutor("vald-index-correction") + err = exe.CreateAndWait(t, ctx, "correction-test") + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + // check if replica number is correct + err = op.StreamListObject(t, ctx, operation.Dataset{ + Train: train, + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + t.Log("Test case 2: execute index correction after one agent removed") + t.Log("removing vald-agent-ngt-0...") + cmd := exec.CommandContext(ctx, "sh", "-c", "kubectl delete pod vald-agent-ngt-0 && kubectl wait --for=condition=Ready pod/vald-agent-ngt-0") + out, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + t.Fatalf("%s, %s, %v", string(out), string(exitErr.Stderr), err) + } else { + t.Fatalf("unexpected error on creating job: %v", err) + } + } + t.Log(string(out)) + + // correct the deleted index + err = exe.CreateAndWait(t, ctx, "correction-test") + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + // check if replica number is correct + err = op.StreamListObject(t, ctx, operation.Dataset{ + Train: train, + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } + + t.Log("Tear down. Removing all vectors...") + err = op.Remove(t, ctx, operation.Dataset{ + Train: train, + }) + if err != nil { + t.Fatalf("an error occurred: %s", err) + } +} diff --git a/tests/e2e/operation/job.go b/tests/e2e/operation/job.go new file mode 100644 index 0000000000..20b20fb17d --- /dev/null +++ b/tests/e2e/operation/job.go @@ -0,0 +1,121 @@ +//go:build e2e + +// Copyright (C) 2019-2023 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package operation + +import ( + "context" + "fmt" + "os/exec" + "testing" +) + +func (j *cronJobExecute) CreateAndWait(t *testing.T, ctx context.Context, jobName string) error { + if err := createJob(t, jobName, j.cronJob); err != nil { + return err + } + + defer func() { + err := deleteJob(t, jobName) + if err != nil { + t.Errorf("failed to delete job: %s", err) + } + }() + + return waitJob(t, ctx, jobName) +} + +func createJob(t *testing.T, jobName, cronJobName string) error { + t.Helper() + t.Logf("creating job: %s from CronJob %s", jobName, cronJobName) + createCmd := fmt.Sprintf("kubectl create job %s --from=cronjob/%s", jobName, cronJobName) + cmd := exec.Command("sh", "-c", createCmd) + return execCmd(t, cmd) +} + +func deleteJob(t *testing.T, jobName string) error { + t.Helper() + t.Log("deleting correction job") + deleteKubeCmd := fmt.Sprintf("kubectl delete job %s", jobName) + cmd := exec.Command("sh", "-c", deleteKubeCmd) + return execCmd(t, cmd) +} + +func waitJob(t *testing.T, ctx context.Context, jobName string) error { + t.Helper() + t.Log("waiting for the correction job to complete or fail") + waitCompleteCmd := fmt.Sprintf("kubectl wait --timeout=10m job/%s --for=condition=complete", jobName) + waitFailedCmd := fmt.Sprintf("kubectl wait --timeout=10m job/%s --for=condition=failed", jobName) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + complete := make(chan struct{}) + failed := make(chan struct{}) + ech := make(chan error) + go func() { + cmd := exec.CommandContext(ctx, "sh", "-c", waitCompleteCmd) + err := execCmd(t, cmd) + if err != nil { + ech <- err + return + } + + complete <- struct{}{} + }() + + go func() { + cmd := exec.CommandContext(ctx, "sh", "-c", waitFailedCmd) + err := execCmd(t, cmd) + if err != nil { + ech <- err + return + } + + t.Logf("%s failed. dumping status", jobName) + dumpStatusCmd := fmt.Sprintf("kubectl get job %s -o yaml", jobName) + cmd = exec.Command("sh", "-c", dumpStatusCmd) + err = execCmd(t, cmd) + if err != nil { + t.Log("failed to dump status") + } + failed <- struct{}{} + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-complete: + return nil + case <-failed: + return fmt.Errorf("correction job failed") + case err := <-ech: + return err + } +} + +func execCmd(t *testing.T, cmd *exec.Cmd) error { + t.Helper() + out, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + return fmt.Errorf("%s, %s, %w", string(out), string(exitErr.Stderr), err) + } else { + return fmt.Errorf("unexpected error on creating job: %w", err) + } + } + t.Log(string(out)) + return nil +} diff --git a/tests/e2e/operation/operation.go b/tests/e2e/operation/operation.go index 332f8fdaf0..24cc988f7e 100644 --- a/tests/e2e/operation/operation.go +++ b/tests/e2e/operation/operation.go @@ -28,11 +28,6 @@ import ( "google.golang.org/grpc/keepalive" ) -type client struct { - host string - port int -} - type Dataset struct { Train [][]float32 Test [][]float32 @@ -132,15 +127,19 @@ type Client interface { MultiUpsert(t *testing.T, ctx context.Context, ds Dataset) error MultiRemove(t *testing.T, ctx context.Context, ds Dataset) error GetObject(t *testing.T, ctx context.Context, ds Dataset) error + StreamListObject(t *testing.T, ctx context.Context, ds Dataset) error Exists(t *testing.T, ctx context.Context, id string) error CreateIndex(t *testing.T, ctx context.Context) error SaveIndex(t *testing.T, ctx context.Context) error IndexInfo(t *testing.T, ctx context.Context) (*payload.Info_Index_Count, error) } -const ( - defaultSearchTimeout = 4 * int64(time.Second) -) +type client struct { + host string + port int +} + +var _ Client = (*client)(nil) func New(host string, port int) (Client, error) { return &client{ @@ -229,3 +228,19 @@ func (c *client) recall(results []string, neighbors []int) (recall float64) { return recall / float64(len(neighbors)) } + +type JobExecutor interface { + CreateAndWait(t *testing.T, ctx context.Context, jobName string) error +} + +type cronJobExecute struct { + cronJob string +} + +var _ JobExecutor = (*cronJobExecute)(nil) + +func NewCronJobExecutor(cronJob string) JobExecutor { + return &cronJobExecute{ + cronJob: cronJob, + } +} diff --git a/tests/e2e/operation/stream.go b/tests/e2e/operation/stream.go index 585039dc2b..fb6a46ac36 100644 --- a/tests/e2e/operation/stream.go +++ b/tests/e2e/operation/stream.go @@ -17,6 +17,7 @@ package operation import ( "context" + "fmt" "reflect" "strconv" "testing" @@ -1167,3 +1168,63 @@ func (c *client) GetObject( return rerr } + +func (c *client) StreamListObject( + t *testing.T, + ctx context.Context, + ds Dataset, +) error { + t.Log("StreamListObject operation started") + + client, err := c.getClient(ctx) + if err != nil { + return err + } + + sc, err := client.StreamListObject(ctx, &payload.Object_List_Request{}) + if err != nil { + return err + } + + // kv : [indexId]count + indexCnt := make(map[string]int) +exit_loop: + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + res, err := sc.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + break exit_loop + } + return err + } + vec := res.GetVector() + if vec == nil { + st := res.GetStatus() + return fmt.Errorf("returned vector is empty: code: %v, msg: %v, details: %v", st.GetCode(), st.GetMessage(), st.GetDetails()) + } + indexCnt[vec.GetId()]++ + } + } + + if len(indexCnt) != len(ds.Train) { + return fmt.Errorf("the number of vectors returned is different: got %v, want %v", len(indexCnt), len(ds.Train)) + } + + replica := -1 + for k, v := range indexCnt { + if replica == -1 { + replica = v + continue + } + if v != replica { + return fmt.Errorf("the number of vectors returned is different at index id %v: got %v, want %v", k, v, replica) + } + } + + t.Log("StreamListObject operation finished successfully and all vectors are returned with correct replica number") + return nil +} diff --git a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go index 77528a2de0..81ee7cf22d 100644 --- a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go +++ b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go @@ -1,3 +1,5 @@ +//go:build e2e + // // Copyright (C) 2019-2023 vdaas.org vald team // diff --git a/versions/GO_VERSION b/versions/GO_VERSION index 2844977405..bae5c7f667 100644 --- a/versions/GO_VERSION +++ b/versions/GO_VERSION @@ -1 +1 @@ -1.21.1 +1.21.3 diff --git a/versions/HELM_DOCS_VERSION b/versions/HELM_DOCS_VERSION index ca7176690d..0a5af26df3 100644 --- a/versions/HELM_DOCS_VERSION +++ b/versions/HELM_DOCS_VERSION @@ -1 +1 @@ -1.11.2 +1.11.3 diff --git a/versions/HELM_VERSION b/versions/HELM_VERSION index db8770749b..6cd6388360 100644 --- a/versions/HELM_VERSION +++ b/versions/HELM_VERSION @@ -1 +1 @@ -v3.12.3 +v3.13.1 diff --git a/versions/JAEGER_OPERATOR_VERSION b/versions/JAEGER_OPERATOR_VERSION index bb13a7e354..132775c22c 100644 --- a/versions/JAEGER_OPERATOR_VERSION +++ b/versions/JAEGER_OPERATOR_VERSION @@ -1 +1 @@ -2.47.0 +2.49.0 diff --git a/versions/K3S_VERSION b/versions/K3S_VERSION new file mode 100644 index 0000000000..079b9ecbc4 --- /dev/null +++ b/versions/K3S_VERSION @@ -0,0 +1 @@ +v1.28.2-k3s1 diff --git a/versions/OPERATOR_SDK_VERSION b/versions/OPERATOR_SDK_VERSION index f3f644dcf8..181891c459 100644 --- a/versions/OPERATOR_SDK_VERSION +++ b/versions/OPERATOR_SDK_VERSION @@ -1 +1 @@ -v1.31 +v1.32 diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index 4319c95d06..a0c3d81c27 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1 @@ -51.0.3 +51.9.0 diff --git a/versions/PROTOBUF_VERSION b/versions/PROTOBUF_VERSION index 22f742b7da..8830024ad4 100644 --- a/versions/PROTOBUF_VERSION +++ b/versions/PROTOBUF_VERSION @@ -1 +1 @@ -24.3 +24.4 diff --git a/versions/TELEPRESENCE_VERSION b/versions/TELEPRESENCE_VERSION index 3b1fc7950f..0e7079b691 100644 --- a/versions/TELEPRESENCE_VERSION +++ b/versions/TELEPRESENCE_VERSION @@ -1 +1 @@ -2.15.1 +2.16.1 diff --git a/versions/YQ_VERSION b/versions/YQ_VERSION index 5c4432ea28..5ed3936faa 100644 --- a/versions/YQ_VERSION +++ b/versions/YQ_VERSION @@ -1 +1 @@ -v4.35.1 +v4.35.2