Skip to content

Commit

Permalink
Backport #2559 (#2560)
Browse files Browse the repository at this point in the history
Signed-off-by: kpango <kpango@vdaas.org>
kpango authored Aug 2, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 5023899 commit 678e98d
Showing 28 changed files with 561 additions and 72 deletions.
27 changes: 27 additions & 0 deletions .gitfiles
Original file line number Diff line number Diff line change
@@ -76,7 +76,12 @@
.github/workflows/coverage.yml
.github/workflows/detect-internal-config-changes.yml
.github/workflows/dockers-agent-faiss-image.yml
.github/workflows/dockers-agent-faiss-image.yml
.github/workflows/dockers-agent-faiss-image.yml
.github/workflows/dockers-agent-image.yml
.github/workflows/dockers-agent-image.yml
.github/workflows/dockers-agent-ngt-image.yml
.github/workflows/dockers-agent-ngt-image.yml
.github/workflows/dockers-agent-ngt-image.yml
.github/workflows/dockers-agent-sidecar-image.yml
.github/workflows/dockers-benchmark-job-image.yml
@@ -133,6 +138,8 @@ CODE_OF_CONDUCT.md
CONTRIBUTING.md
LICENSE
Makefile
Makefile
Makefile
Makefile.d/actions.mk
Makefile.d/bench.mk
Makefile.d/build.mk
@@ -519,16 +526,23 @@ design/Vald Architecture Dataflow.drawio
design/Vald Architecture Overview.drawio
design/vald_architecture_overview.png
dockers/agent/core/agent/Dockerfile
dockers/agent/core/agent/Dockerfile
dockers/agent/core/agent/README.md
dockers/agent/core/faiss/Dockerfile
dockers/agent/core/faiss/Dockerfile
dockers/agent/core/faiss/Dockerfile
dockers/agent/core/ngt/Dockerfile
dockers/agent/core/ngt/README.md
dockers/agent/sidecar/Dockerfile
dockers/agent/sidecar/README.md
dockers/buildbase/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/README.md
dockers/dev/Dockerfile
dockers/dev/Dockerfile
dockers/dev/Dockerfile
dockers/dev/README.md
dockers/discoverer/k8s/Dockerfile
dockers/discoverer/k8s/README.md
@@ -555,6 +569,7 @@ docs/api/README.md
docs/api/build_proto.md
docs/api/filter-gateway.md
docs/api/flush.md
docs/api/flush.md
docs/api/insert.md
docs/api/mirror-gateway.md
docs/api/object.md
@@ -622,8 +637,13 @@ example/helm/values-with-pyroscope.yaml
example/helm/values.yaml
example/manifest/scylla/configmap.yaml
example/manifest/scylla/job.yaml
github_info.json
go.mod
go.mod
go.mod
go.sum
go.sum
go.sum
hack/CHANGELOG.template.md
hack/benchmark/assets/checksum/fashion-mnist-784-euclidean.md5
hack/benchmark/assets/checksum/gist-960-euclidean.md5
@@ -699,6 +719,7 @@ hack/benchmark/internal/starter/starter.go
hack/benchmark/metrics/metrics.go
hack/benchmark/src/singleflight/singleflight_bench_test.go
hack/docker/gen/main.go
hack/docker/gen/main.go
hack/git/hooks/pre-commit
hack/go.mod.default
hack/gorules/rules.go
@@ -1936,8 +1957,12 @@ pkg/tools/cli/loadtest/usecase/load.go
pkg/tools/cli/loadtest/usecase/load_test.go
renovate.json
rust/Cargo.lock
rust/Cargo.lock
rust/Cargo.lock
rust/Cargo.toml
rust/bin/agent/Cargo.toml
rust/bin/agent/Cargo.toml
rust/bin/agent/Cargo.toml
rust/bin/agent/src/handler.rs
rust/bin/agent/src/handler/common.rs
rust/bin/agent/src/handler/index.rs
@@ -2019,6 +2044,8 @@ versions/NGT_VERSION
versions/OPERATOR_SDK_VERSION
versions/OTEL_OPERATOR_VERSION
versions/PROMETHEUS_STACK_VERSION
versions/PROMETHEUS_STACK_VERSION
versions/PROMETHEUS_STACK_VERSION
versions/PROTOBUF_VERSION
versions/REVIEWDOG_VERSION
versions/RUST_VERSION
31 changes: 16 additions & 15 deletions .github/workflows/dockers-agent-faiss-image.yml
Original file line number Diff line number Diff line change
@@ -27,42 +27,43 @@ on:
- "v*.*.*-*"
pull_request:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-faiss-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/faiss/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "versions/GO_VERSION"
- "versions/FAISS_VERSION"
- "versions/GO_VERSION"
pull_request_target:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-faiss-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/faiss/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "versions/GO_VERSION"
- "versions/FAISS_VERSION"
- "versions/GO_VERSION"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
with:
target: agent-faiss
platforms: linux/amd64
secrets: inherit
16 changes: 10 additions & 6 deletions .github/workflows/dockers-agent-image.yml
Original file line number Diff line number Diff line change
@@ -30,29 +30,33 @@ on:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-image.yml"
- "Makefile"
- "dockers/agent/core/agent/Dockerfile"
- "rust/Cargo.lock"
- "rust/Cargo.toml"
- "rust/bin/agent/**"
- "rust/libs/ngt/**"
- "rust/libs/ngt-rs/**"
- "rust/libs/ngt/**"
- "rust/libs/proto/**"
- "dockers/agent/core/agent/Dockerfile"
- "versions/RUST_VERSION"
- "versions/FAISS_VERSION"
- "versions/NGT_VERSION"
- "versions/RUST_VERSION"
pull_request_target:
paths:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-image.yml"
- "Makefile"
- "dockers/agent/core/agent/Dockerfile"
- "rust/Cargo.lock"
- "rust/Cargo.toml"
- "rust/bin/agent/**"
- "rust/libs/ngt/**"
- "rust/libs/ngt-rs/**"
- "rust/libs/ngt/**"
- "rust/libs/proto/**"
- "dockers/agent/core/agent/Dockerfile"
- "versions/RUST_VERSION"
- "versions/FAISS_VERSION"
- "versions/NGT_VERSION"
- "versions/RUST_VERSION"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
30 changes: 16 additions & 14 deletions .github/workflows/dockers-agent-ngt-image.yml
Original file line number Diff line number Diff line change
@@ -27,38 +27,40 @@ on:
- "v*.*.*-*"
pull_request:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-ngt-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/ngt/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "versions/GO_VERSION"
- "versions/NGT_VERSION"
- "versions/GO_VERSION"
pull_request_target:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-ngt-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/ngt/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "versions/GO_VERSION"
- "versions/NGT_VERSION"
- "versions/GO_VERSION"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
64 changes: 64 additions & 0 deletions .github/workflows/update-actions.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Update Actions version
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
jobs:
dump-contexts-to-log:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: vdaas/vald/.github/actions/dump-context@main
update-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.DISPATCH_TOKEN }}
- name: Set Git config
run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE}
- uses: crazy-max/ghaction-import-gpg@v6
with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
git_user_signingkey: true
git_commit_gpgsign: true
- name: Run update command
run: |
make update/actions
- name: Check difference
id: check_diff
run: |
if git diff --quiet --exit-code; then
echo "Nothing updated"
else
git diff && git status
echo "HAS_GIT_DIFF=true" >> $GITHUB_OUTPUT
fi
- name: Create PR
if: ${{ steps.check_diff.outputs.HAS_GIT_DIFF == 'true' }}
uses: peter-evans/create-pull-request@v6
with:
author: "${{ secrets.DISPATCH_USER }} <vald@vdaas.org>"
token: ${{ secrets.DISPATCH_TOKEN }}
committer: "${{ secrets.DISPATCH_USER }} <vald@vdaas.org>"
signoff: true
delete-branch: true
base: main
title: "Update Actions dependency"
body: "Automated pull request to update Actions."
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@
# limitations under the License.
#

SHELL = bash
ORG ?= vdaas
NAME = vald
GOPKG = github.com/$(ORG)/$(NAME)
@@ -144,7 +145,7 @@ PBDOCS = apis/docs/v1/docs.md
LDFLAGS = -static -fPIC -pthread -std=gnu++23 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections

NGT_LDFLAGS = -fopenmp -lopenblas -llapack
FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran -lquadmath
FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran
HDF5_LDFLAGS = -lhdf5 -lhdf5_hl -lsz -laec -lz -ldl
CGO_LDFLAGS = $(FAISS_LDFLAGS) $(HDF5_LDFLAGS)

3 changes: 1 addition & 2 deletions dockers/agent/core/agent/Dockerfile
Original file line number Diff line number Diff line change
@@ -43,8 +43,8 @@ ENV REPO=vald
ENV RUST_HOME=/usr/loacl/lib/rust
ENV TZ=Etc/UTC
ENV USER=root
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV CARGO_HOME=${RUST_HOME}/cargo
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}

WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO}
@@ -75,7 +75,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
&& ldconfig \
&& echo "${LANG} UTF-8" > /etc/locale.gen \
&& ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
1 change: 0 additions & 1 deletion dockers/agent/core/faiss/Dockerfile
Original file line number Diff line number Diff line change
@@ -77,7 +77,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
&& ldconfig \
&& echo "${LANG} UTF-8" > /etc/locale.gen \
&& ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
3 changes: 1 addition & 2 deletions dockers/ci/base/Dockerfile
Original file line number Diff line number Diff line change
@@ -48,8 +48,8 @@ ENV REPO=vald
ENV RUST_HOME=/usr/loacl/lib/rust
ENV TZ=Etc/UTC
ENV USER=root
ENV CARGO_HOME=${RUST_HOME}/cargo
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV CARGO_HOME=${RUST_HOME}/cargo
ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}

WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
@@ -83,7 +83,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
gawk \
gnupg2 \
graphviz \
1 change: 0 additions & 1 deletion dockers/dev/Dockerfile
Original file line number Diff line number Diff line change
@@ -88,7 +88,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
gawk \
gnupg2 \
graphviz \
15 changes: 7 additions & 8 deletions docs/api/flush.md
Original file line number Diff line number Diff line change
@@ -31,7 +31,6 @@ Flush RPC is the method to remove all vectors.
empty
### Output
- the scheme of `payload.v1.Info.Index.Count`
@@ -47,14 +46,14 @@ Flush RPC is the method to remove all vectors.
}
```

Object.Info_Index_Count
Object.Info_Index_Count

| field | type | label | desc. |
| :---------: |:------ | :---- | :------------------------------------------------------------------------- |
| stored | uint32 | | count of indices. |
| uncommitted | uint32 | | count of uncommitted indices. |
| indexing | bool | | the state indicating whether `vald-agent` pods is present in the indexing. |
| saving | bool | | the state indicating whether `vald-agent` pods is present in the saving. |
| field | type | label | desc. |
| :---------: | :----- | :---- | :------------------------------------------------------------------------- |
| stored | uint32 | | count of indices. |
| uncommitted | uint32 | | count of uncommitted indices. |
| indexing | bool | | the state indicating whether `vald-agent` pods is present in the indexing. |
| saving | bool | | the state indicating whether `vald-agent` pods is present in the saving. |

### Status Code

10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
@@ -314,7 +314,7 @@ replace (
gonum.org/v1/gonum => gonum.org/v1/gonum v0.15.0
gonum.org/v1/hdf5 => gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946
gonum.org/v1/plot => gonum.org/v1/plot v0.14.0
google.golang.org/api => google.golang.org/api v0.189.0
google.golang.org/api => google.golang.org/api v0.190.0
google.golang.org/appengine => google.golang.org/appengine v1.6.8
google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf
google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf
@@ -406,7 +406,7 @@ require (
gonum.org/v1/hdf5 v0.0.0-00010101000000-000000000000
gonum.org/v1/plot v0.14.0
google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f
google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf
google.golang.org/grpc v1.65.0
google.golang.org/protobuf v1.34.2
gopkg.in/yaml.v2 v2.4.0
@@ -422,7 +422,7 @@ require (

require (
cloud.google.com/go v0.115.0 // indirect
cloud.google.com/go/auth v0.7.2 // indirect
cloud.google.com/go/auth v0.7.3 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go/iam v1.1.12 // indirect
@@ -462,7 +462,7 @@ require (
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/wire v0.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
@@ -514,7 +514,7 @@ require (
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/api v0.189.0 // indirect
google.golang.org/genproto v0.0.0-20240725223205-93522f1f2a9f // indirect
google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.30.1 // indirect
14 changes: 9 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
@@ -24,8 +24,9 @@ cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbw
cloud.google.com/go/auth v0.6.0/go.mod h1:b4acV+jLQDyjwm4OXHYjNvRi4jvGBzHWJRtJcy+2P4g=
cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4=
cloud.google.com/go/auth v0.7.0/go.mod h1:D+WqdrpcjmiCgWrXmLLxOVq1GACoE36chW6KXoEvuIw=
cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE=
cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs=
cloud.google.com/go/auth v0.7.3 h1:98Vr+5jMaCZ5NZk6e/uBgf60phTk/XN84r8QEWB9yjY=
cloud.google.com/go/auth v0.7.3/go.mod h1:HJtWUx1P5eqjy/f6Iq5KeytNpbAcGolPhOgyop2LlzA=
cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI=
cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I=
@@ -96,6 +97,7 @@ cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFV
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc=
cloud.google.com/go/longrunning v0.5.2/go.mod h1:nqo6DQbNV2pXhGDbDMoN2bWz68MjZUzqv2YttZiveCs=
cloud.google.com/go/longrunning v0.5.6/go.mod h1:vUaDrWYOMKRuhiv6JBnn49YxCPz2Ayn9GqyjaBT8/mA=
cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng=
cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c=
cloud.google.com/go/longrunning v0.5.10/go.mod h1:tljz5guTr5oc/qhlUjBlk7UAIFMOGuPNxkNDZXlLics=
@@ -146,6 +148,7 @@ cloud.google.com/go/talent v1.6.12/go.mod h1:nT9kNVuJhZX2QgqKZS6t6eCWZs5XEBYRBv6
cloud.google.com/go/texttospeech v1.7.11/go.mod h1:Ua125HU+WT2IkIo5MzQtuNpNEk72soShJQVdorZ1SAE=
cloud.google.com/go/tpu v1.6.11/go.mod h1:W0C4xaSj1Ay3VX/H96FRvLt2HDs0CgdRPVI4e7PoCDk=
cloud.google.com/go/trace v1.10.11/go.mod h1:fUr5L3wSXerNfT0f1bBg08W4axS2VbHGgYcfH4KuTXU=
cloud.google.com/go/translate v1.10.3/go.mod h1:GW0vC1qvPtd3pgtypCv4k4U8B7EdgK9/QEF2aJEUovs=
cloud.google.com/go/translate v1.10.7/go.mod h1:mH/+8tvcItuy1cOWqU+/Y3iFHgkVUObNIQYI/kiFFiY=
cloud.google.com/go/video v1.22.0/go.mod h1:CxPshUNAb1ucnzbtruEHlAal9XY+SPG2cFqC/woJzII=
cloud.google.com/go/videointelligence v1.11.11/go.mod h1:dab2Ca3AXT6vNJmt3/6ieuquYRckpsActDekLcsd6dU=
@@ -411,8 +414,9 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
@@ -721,14 +725,14 @@ gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENE
gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I=
gonum.org/v1/plot v0.14.0 h1:+LBDVFYwFe4LHhdP8coW6296MBEY4nQ+Y4vuUpJopcE=
gonum.org/v1/plot v0.14.0/go.mod h1:MLdR9424SJed+5VqC6MsouEpig9pZX2VZ57H9ko2bXU=
google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI=
google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8=
google.golang.org/api v0.190.0 h1:ASM+IhLY1zljNdLu19W1jTmU6A+gMk6M46Wlur61s+Q=
google.golang.org/api v0.190.0/go.mod h1:QIr6I9iedBLnfqoD6L6Vze1UvS5Hzj5r2aUBOaZnLHo=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf h1:OqdXDEakZCVtDiZTjcxfwbHPCT11ycCEsTKesBVKvyY=
google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:mCr1K1c8kX+1iSBREvU3Juo11CB+QOEWxbRS01wWl5M=
google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf h1:GillM0Ef0pkZPIB+5iO6SDK+4T9pf6TpaYR6ICD5rVE=
google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:OFMYQFHJ4TM3JRlWDZhJbZfra2uqc3WLBZiaaqP4DtU=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20240722135656-d784300faade/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
1 change: 0 additions & 1 deletion hack/docker/gen/main.go
Original file line number Diff line number Diff line change
@@ -348,7 +348,6 @@ var (
}
faissBuildDeps = []string{
"gfortran",
"libquadmath0",
}
devContainerDeps = []string{
"gawk",
4 changes: 2 additions & 2 deletions rust/Cargo.lock
5 changes: 5 additions & 0 deletions rust/bin/agent/Cargo.toml
Original file line number Diff line number Diff line change
@@ -22,3 +22,8 @@ edition = "2021"

[dependencies]
ngt = { version = "0.1.0", path = "../../libs/ngt" }
prost = "0.12.4"
proto = { version = "0.1.0", path = "../../libs/proto" }
tokio = { version = "1.37.0", features = ["full"] }
tokio-stream = { version = "0.1.15", features = ["full"] }
tonic = "0.11.0"
27 changes: 27 additions & 0 deletions rust/bin/agent/src/handler.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
//
// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
mod common;
mod index;
mod insert;
mod remove;
mod search;
mod update;
mod upsert;

#[derive(Default, Debug)]
pub struct Agent {

}
21 changes: 21 additions & 0 deletions rust/bin/agent/src/handler/common.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
//
// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#[macro_export]
macro_rules! stream_type {
($t:ty) => {
std::pin::Pin<Box<dyn tokio_stream::Stream<Item = std::result::Result<$t, tonic::Status>> + Send>>
};
}
47 changes: 47 additions & 0 deletions rust/bin/agent/src/handler/insert.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
//
// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
use proto::{
payload::v1::{insert, object},
vald::v1::insert_server,
};
#[tonic::async_trait]
impl insert_server::Insert for super::Agent {
async fn insert(
&self,
request: tonic::Request<insert::Request>,
) -> std::result::Result<tonic::Response<object::Location>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamInsert method."]
type StreamInsertStream = crate::stream_type!(object::StreamLocation);

#[doc = " A method to add new multiple vectors by bidirectional streaming.\n"]
async fn stream_insert(
&self,
request: tonic::Request<tonic::Streaming<insert::Request>>,
) -> std::result::Result<tonic::Response<Self::StreamInsertStream>, tonic::Status> {
todo!()
}

#[doc = " A method to add new multiple vectors in a single request.\n"]
async fn multi_insert(
&self,
request: tonic::Request<insert::MultiRequest>,
) -> std::result::Result<tonic::Response<object::Locations>, tonic::Status> {
todo!()
}
}
56 changes: 56 additions & 0 deletions rust/bin/agent/src/handler/remove.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
//
// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
use proto::{
payload::v1::{object, remove},
vald::v1::remove_server,
};

#[tonic::async_trait]
impl remove_server::Remove for super::Agent {
async fn remove(
&self,
request: tonic::Request<remove::Request>,
) -> std::result::Result<tonic::Response<object::Location>, tonic::Status> {
todo!()
}

#[doc = " A method to remove an indexed vector based on timestamp.\n"]
async fn remove_by_timestamp(
&self,
request: tonic::Request<remove::TimestampRequest>,
) -> std::result::Result<tonic::Response<object::Locations>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamRemove method."]
type StreamRemoveStream = crate::stream_type!(object::StreamLocation);

#[doc = " A method to remove multiple indexed vectors by bidirectional streaming.\n"]
async fn stream_remove(
&self,
request: tonic::Request<tonic::Streaming<remove::Request>>,
) -> std::result::Result<tonic::Response<Self::StreamRemoveStream>, tonic::Status> {
todo!()
}

#[doc = " A method to remove multiple indexed vectors in a single request.\n"]
async fn multi_remove(
&self,
request: tonic::Request<remove::MultiRequest>,
) -> std::result::Result<tonic::Response<object::Locations>, tonic::Status> {
todo!()
}
}
127 changes: 127 additions & 0 deletions rust/bin/agent/src/handler/search.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
//
// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
use proto::{payload::v1::search, vald::v1::search_server};

#[tonic::async_trait]
impl search_server::Search for super::Agent {
async fn search(
&self,
request: tonic::Request<search::Request>,
) -> Result<tonic::Response<search::Response>, tonic::Status> {
todo!()
}

#[doc = " A method to search indexed vectors by ID.\n"]
async fn search_by_id(
&self,
request: tonic::Request<search::IdRequest>,
) -> Result<tonic::Response<search::Response>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamSearch method."]
type StreamSearchStream = crate::stream_type!(search::StreamResponse);

#[doc = " A method to search indexed vectors by multiple vectors.\n"]
async fn stream_search(
&self,
request: tonic::Request<tonic::Streaming<search::Request>>,
) -> std::result::Result<tonic::Response<Self::StreamSearchStream>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamSearchByID method."]
type StreamSearchByIDStream = crate::stream_type!(search::StreamResponse);

#[doc = " A method to search indexed vectors by multiple IDs.\n"]
async fn stream_search_by_id(
&self,
request: tonic::Request<tonic::Streaming<search::IdRequest>>,
) -> std::result::Result<tonic::Response<Self::StreamSearchByIDStream>, tonic::Status> {
todo!()
}

#[doc = " A method to search indexed vectors by multiple vectors in a single request.\n"]
async fn multi_search(
&self,
request: tonic::Request<search::MultiRequest>,
) -> std::result::Result<tonic::Response<search::Responses>, tonic::Status> {
todo!()
}

#[doc = " A method to search indexed vectors by multiple IDs in a single request.\n"]
async fn multi_search_by_id(
&self,
request: tonic::Request<search::MultiIdRequest>,
) -> std::result::Result<tonic::Response<search::Responses>, tonic::Status> {
todo!()
}

#[doc = " A method to linear search indexed vectors by a raw vector.\n"]
async fn linear_search(
&self,
request: tonic::Request<search::Request>,
) -> std::result::Result<tonic::Response<search::Response>, tonic::Status> {
todo!()
}

#[doc = " A method to linear search indexed vectors by ID.\n"]
async fn linear_search_by_id(
&self,
request: tonic::Request<search::IdRequest>,
) -> std::result::Result<tonic::Response<search::Response>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamLinearSearch method."]
type StreamLinearSearchStream = crate::stream_type!(search::StreamResponse);

#[doc = " A method to linear search indexed vectors by multiple vectors.\n"]
async fn stream_linear_search(
&self,
request: tonic::Request<tonic::Streaming<search::Request>>,
) -> std::result::Result<tonic::Response<Self::StreamLinearSearchStream>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamLinearSearchByID method."]
type StreamLinearSearchByIDStream = crate::stream_type!(search::StreamResponse);

#[doc = " A method to linear search indexed vectors by multiple IDs.\n"]
async fn stream_linear_search_by_id(
&self,
request: tonic::Request<tonic::Streaming<search::IdRequest>>,
) -> std::result::Result<tonic::Response<Self::StreamLinearSearchByIDStream>, tonic::Status>
{
todo!()
}

#[doc = " A method to linear search indexed vectors by multiple vectors in a single\n request.\n"]
async fn multi_linear_search(
&self,
request: tonic::Request<search::MultiRequest>,
) -> std::result::Result<tonic::Response<search::Responses>, tonic::Status> {
todo!()
}

#[doc = " A method to linear search indexed vectors by multiple IDs in a single\n request.\n"]
async fn multi_linear_search_by_id(
&self,
request: tonic::Request<search::MultiIdRequest>,
) -> std::result::Result<tonic::Response<search::Responses>, tonic::Status> {
todo!()
}
}
48 changes: 48 additions & 0 deletions rust/bin/agent/src/handler/update.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
//
// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
use proto::{
payload::v1::{object, update},
vald::v1::update_server,
};

#[tonic::async_trait]
impl update_server::Update for super::Agent {
async fn update(
&self,
request: tonic::Request<update::Request>,
) -> std::result::Result<tonic::Response<object::Location>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamUpdate method."]
type StreamUpdateStream = crate::stream_type!(object::StreamLocation);

#[doc = " A method to update multiple indexed vectors by bidirectional streaming.\n"]
async fn stream_update(
&self,
request: tonic::Request<tonic::Streaming<update::Request>>,
) -> std::result::Result<tonic::Response<Self::StreamUpdateStream>, tonic::Status> {
todo!()
}

#[doc = " A method to update multiple indexed vectors in a single request.\n"]
async fn multi_update(
&self,
request: tonic::Request<update::MultiRequest>,
) -> std::result::Result<tonic::Response<object::Locations>, tonic::Status> {
todo!()
}
}
48 changes: 48 additions & 0 deletions rust/bin/agent/src/handler/upsert.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
//
// Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
use proto::{
payload::v1::{object, upsert},
vald::v1::upsert_server,
};

#[tonic::async_trait]
impl upsert_server::Upsert for super::Agent {
async fn upsert(
&self,
request: tonic::Request<upsert::Request>,
) -> std::result::Result<tonic::Response<object::Location>, tonic::Status> {
todo!()
}

#[doc = " Server streaming response type for the StreamUpsert method."]
type StreamUpsertStream = crate::stream_type!(object::StreamLocation);

#[doc = " A method to insert/update multiple vectors by bidirectional streaming.\n"]
async fn stream_upsert(
&self,
request: tonic::Request<tonic::Streaming<upsert::Request>>,
) -> std::result::Result<tonic::Response<Self::StreamUpsertStream>, tonic::Status> {
todo!()
}

#[doc = " A method to insert/update multiple vectors in a single request.\n"]
async fn multi_upsert(
&self,
request: tonic::Request<upsert::MultiRequest>,
) -> std::result::Result<tonic::Response<object::Locations>, tonic::Status> {
todo!()
}
}
16 changes: 14 additions & 2 deletions rust/bin/agent/src/main.rs
Original file line number Diff line number Diff line change
@@ -13,6 +13,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
fn main() {
println!("Hello, world!");

mod handler;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let addr = "[::1]:8081".parse()?;
let agent = handler::Agent::default();

tonic::transport::Server::builder()
.add_service(proto::core::v1::agent_server::AgentServer::new(agent))
.serve(addr)
.await?;

Ok(())
}
4 changes: 2 additions & 2 deletions rust/libs/proto/Cargo.toml
Original file line number Diff line number Diff line change
@@ -23,5 +23,5 @@ edition = "2021"
[dependencies]
futures-core = "0.3.30"
prost = "0.12.3"
tonic = "0.10.2"
tonic-types = "0.10.2"
tonic = "0.11.0"
tonic-types = "0.11.0"
8 changes: 4 additions & 4 deletions rust/libs/proto/src/lib.rs
Original file line number Diff line number Diff line change
@@ -31,8 +31,8 @@ pub mod vald {
}
}

#[cfg(test)]
mod tests {
#[test]
fn it_works() {}
pub mod core {
pub mod v1 {
include!("core.v1.tonic.rs");
}
}
2 changes: 1 addition & 1 deletion versions/PROMETHEUS_STACK_VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
61.6.0
61.6.1
1 change: 1 addition & 0 deletions versions/actions/PETER_EVANS_CREATE_PULL_REQUEST
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6.1.0

0 comments on commit 678e98d

Please sign in to comment.