Add UpdateTimestamp API

Signed-off-by: kpango <[email protected]>
vdaas · Sep 3, 2024 · debe3d2 · debe3d2
1 parent b17cc73
commit debe3d2
Show file tree

Hide file tree

Showing 164 changed files with 6,929 additions and 3,178 deletions.
diff --git a/.cspell.json b/.cspell.json
diff --git a/.gitfiles b/.gitfiles
@@ -140,7 +140,6 @@ Makefile
 Makefile.d/actions.mk
 Makefile.d/bench.mk
 Makefile.d/build.mk
-Makefile.d/client.mk
 Makefile.d/dependencies.mk
 Makefile.d/docker.mk
 Makefile.d/e2e.mk
@@ -511,6 +510,7 @@ cmd/tools/cli/loadtest/main_test.go
 cmd/tools/cli/loadtest/sample.yaml
 cmd/tools/cli/vdctl/main.go
 cmd/tools/cli/vdctl/main_test.go
+codecov.yml
 design/Vald Architecture Assets.drawio
 design/Vald Architecture Dataflow.drawio
 design/Vald Architecture Overview.drawio
@@ -698,6 +698,7 @@ hack/benchmark/internal/starter/gateway/vald/vald_test.go
 hack/benchmark/internal/starter/starter.go
 hack/benchmark/metrics/metrics.go
 hack/benchmark/src/singleflight/singleflight_bench_test.go
+hack/cspell/main.go
 hack/docker/gen/main.go
 hack/git/hooks/pre-commit
 hack/go.mod.default
@@ -1594,6 +1595,7 @@ pkg/agent/internal/kvs/kvs.go
 pkg/agent/internal/kvs/kvs_test.go
 pkg/agent/internal/kvs/option.go
 pkg/agent/internal/kvs/option_test.go
+pkg/agent/internal/memstore/data_manager.go
 pkg/agent/internal/metadata/metadata.go
 pkg/agent/internal/metadata/metadata_test.go
 pkg/agent/internal/vqueue/option.go
@@ -1897,13 +1899,20 @@ rust/bin/agent/src/handler/search.rs
 rust/bin/agent/src/handler/update.rs
 rust/bin/agent/src/handler/upsert.rs
 rust/bin/agent/src/main.rs
-rust/libs/ngt-rs/Cargo.toml
-rust/libs/ngt-rs/build.rs
-rust/libs/ngt-rs/src/input.cpp
-rust/libs/ngt-rs/src/input.h
-rust/libs/ngt-rs/src/lib.rs
-rust/libs/ngt/Cargo.toml
-rust/libs/ngt/src/lib.rs
+rust/libs/algorithm/Cargo.toml
+rust/libs/algorithm/src/lib.rs
+rust/libs/algorithms/faiss/Cargo.toml
+rust/libs/algorithms/faiss/src/lib.rs
+rust/libs/algorithms/ngt/Cargo.toml
+rust/libs/algorithms/ngt/build.rs
+rust/libs/algorithms/ngt/src/input.cpp
+rust/libs/algorithms/ngt/src/input.h
+rust/libs/algorithms/ngt/src/lib.rs
+rust/libs/observability/Cargo.toml
+rust/libs/observability/src/config.rs
+rust/libs/observability/src/lib.rs
+rust/libs/observability/src/macros.rs
+rust/libs/observability/src/observability.rs
 rust/libs/proto/Cargo.toml
 rust/libs/proto/src/core.v1.tonic.rs
 rust/libs/proto/src/discoverer.v1.tonic.rs
@@ -1965,7 +1974,6 @@ versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
 versions/RUST_VERSION
 versions/TELEPRESENCE_VERSION
-versions/VALDCLI_VERSION
 versions/VALD_VERSION
 versions/YQ_VERSION
 versions/ZLIB_VERSION

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -25,8 +25,8 @@ assignees: ""
 - Vald Version: v1.7.13
 - Go Version: v1.23.0
 - Rust Version: v1.80.0
-- Docker Version: v27.1.1
-- Kubernetes Version: v1.30.3
-- Helm Version: v3.15.3
+- Docker Version: v27.2.0
+- Kubernetes Version: v1.31.0
+- Helm Version: v3.15.4
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
diff --git a/.github/ISSUE_TEMPLATE/security_issue_report.md b/.github/ISSUE_TEMPLATE/security_issue_report.md
@@ -19,8 +19,8 @@ assignees: ""
 - Vald Version: v1.7.13
 - Go Version: v1.23.0
 - Rust Version: v1.80.0
-- Docker Version: v27.1.1
-- Kubernetes Version: v1.30.3
-- Helm Version: v3.15.3
+- Docker Version: v27.2.0
+- Kubernetes Version: v1.31.0
+- Helm Version: v3.15.4
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -18,9 +18,9 @@
 - Vald Version: v1.7.13
 - Go Version: v1.23.0
 - Rust Version: v1.80.0
-- Docker Version: v27.1.1
-- Kubernetes Version: v1.30.3
-- Helm Version: v3.15.3
+- Docker Version: v27.2.0
+- Kubernetes Version: v1.31.0
+- Helm Version: v3.15.4
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
 

diff --git a/.github/helm/values/values-correction.yaml b/.github/helm/values/values-correction.yaml
@@ -16,7 +16,7 @@
 
 defaults:
   logging:
-    level: info
+    level: debug
   networkPolicy:
     enabled: true
 gateway:

diff --git a/.github/workflows/dockers-benchmark-job-image.yml b/.github/workflows/dockers-benchmark-job-image.yml
@@ -27,7 +27,7 @@ on:
     paths:
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
-      - ".github/workflows/dockers-benchmak-job-image.yml"
+      - ".github/workflows/dockers-benchmark-job-image.yml"
       - "go.mod"
       - "go.sum"
       - "internal/**"
@@ -44,7 +44,7 @@ on:
     paths:
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
-      - ".github/workflows/dockers-benchmak-job-image.yml"
+      - ".github/workflows/dockers-benchmark-job-image.yml"
       - "go.mod"
       - "go.sum"
       - "internal/**"

diff --git a/.github/workflows/dockers-benchmark-operator-image.yaml b/.github/workflows/dockers-benchmark-operator-image.yaml
@@ -27,7 +27,7 @@ on:
     paths:
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
-      - ".github/workflows/dockers-benchmak-operator-image.yml"
+      - ".github/workflows/dockers-benchmark-operator-image.yml"
       - "go.mod"
       - "go.sum"
       - "internal/**"
@@ -44,7 +44,7 @@ on:
     paths:
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
-      - ".github/workflows/dockers-benchmak-operator-image.yml"
+      - ".github/workflows/dockers-benchmark-operator-image.yml"
       - "go.mod"
       - "go.sum"
       - "internal/**"

diff --git a/.github/workflows/update-actions.yaml b/.github/workflows/update-actions.yaml
@@ -52,7 +52,7 @@ jobs:
           fi
       - name: Create PR
         if: ${{ steps.check_diff.outputs.HAS_GIT_DIFF == 'true' }}
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
           author: "${{ secrets.DISPATCH_USER }} <[email protected]>"
           token: ${{ secrets.DISPATCH_TOKEN }}

diff --git a/Makefile b/Makefile
@@ -103,7 +103,6 @@ PROTOBUF_VERSION          := $(eval PROTOBUF_VERSION := $(shell cat versions/PRO
 REVIEWDOG_VERSION         := $(eval REVIEWDOG_VERSION := $(shell cat versions/REVIEWDOG_VERSION))$(REVIEWDOG_VERSION)
 RUST_VERSION              := $(eval RUST_VERSION := $(shell cat versions/RUST_VERSION))$(RUST_VERSION)
 TELEPRESENCE_VERSION      := $(eval TELEPRESENCE_VERSION := $(shell cat versions/TELEPRESENCE_VERSION))$(TELEPRESENCE_VERSION)
-VALDCLI_VERSION           := $(eval VALDCLI_VERSION := $(shell cat versions/VALDCLI_VERSION))$(VALDCLI_VERSION)
 YQ_VERSION                := $(eval YQ_VERSION := $(shell cat versions/YQ_VERSION))$(YQ_VERSION)
 ZLIB_VERSION              := $(eval ZLIB_VERSION := $(shell cat versions/ZLIB_VERSION))$(ZLIB_VERSION)
 
@@ -455,7 +454,6 @@ init: \
 tools/install: \
 	helm/install \
 	kind/install \
-	valdcli/install \
 	telepresence/install \
 	textlint/install
 
@@ -620,10 +618,6 @@ version/helm:
 version/yq:
 	@echo $(YQ_VERSION)
 
-.PHONY: version/valdcli
-version/valdcli:
-	@echo $(VALDCLI_VERSION)
-
 .PHONY: version/telepresence
 version/telepresence:
 	@echo $(TELEPRESENCE_VERSION)
@@ -737,14 +731,14 @@ files/textlint: \
 ## run cspell for document
 docs/cspell:\
 	cspell/install
-	cspell-cli $(ROOTDIR)/docs/**/*.md --show-suggestions $(CSPELL_EXTRA_OPTIONS)
+	cspell $(ROOTDIR)/docs/**/*.md --show-suggestions $(CSPELL_EXTRA_OPTIONS)
 
 .PHONY: files/cspell
 ## run cspell for document
 files/cspell: \
 	files \
 	cspell/install
-	cspell-cli $(ROOTDIR)/.gitfiles --show-suggestions $(CSPELL_EXTRA_OPTIONS)
+	cspell $(ROOTDIR)/.gitfiles --show-suggestions $(CSPELL_EXTRA_OPTIONS)
 
 .PHONY: changelog/update
 ## update changelog
@@ -766,7 +760,6 @@ changelog/next/print:
 include Makefile.d/actions.mk
 include Makefile.d/bench.mk
 include Makefile.d/build.mk
-include Makefile.d/client.mk
 include Makefile.d/dependencies.mk
 include Makefile.d/docker.mk
 include Makefile.d/e2e.mk
@@ -775,7 +768,7 @@ include Makefile.d/helm.mk
 include Makefile.d/k3d.mk
 include Makefile.d/k8s.mk
 include Makefile.d/kind.mk
+include Makefile.d/minikube.mk
 include Makefile.d/proto.mk
 include Makefile.d/test.mk
 include Makefile.d/tools.mk
-include Makefile.d/minikube.mk
diff --git a/Makefile.d/client.mk b/Makefile.d/client.mk
diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk
@@ -39,7 +39,6 @@ update/libs: \
 	update/rust \
 	update/telepresence \
 	update/vald \
-	update/valdcli \
 	update/yq \
 	update/zlib
 
@@ -227,11 +226,6 @@ update/hdf5:
 update/vald:
 	curl -fsSL https://api.github.com/repos/$(REPO)/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALD_VERSION
 
-.PHONY: update/valdcli
-## update vald client library made by clojure self version
-update/valdcli:
-	curl -fsSL https://api.github.com/repos/$(REPO)-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION
-
 .PHONY: update/template
 ## update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE
 update/template:

diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk
@@ -41,6 +41,33 @@ docker/build: \
 	docker/build/operator/helm \
 	docker/build/readreplica-rotate
 
+docker/xpanes/build:
+	@xpanes -s -c "make -f $(ROOTDIR)/Makefile {}" \
+		docker/build/agent \
+		docker/build/agent-faiss \
+		docker/build/agent-ngt \
+		docker/build/agent-sidecar \
+		docker/build/benchmark-job \
+		docker/build/benchmark-operator \
+		docker/build/binfmt \
+		docker/build/buildbase \
+		docker/build/buildkit \
+		docker/build/buildkit-syft-scanner \
+		docker/build/ci-container \
+		docker/build/dev-container \
+		docker/build/discoverer-k8s \
+		docker/build/gateway-filter \
+		docker/build/gateway-lb \
+		docker/build/gateway-mirror \
+		docker/build/index-correction \
+		docker/build/index-creation \
+		docker/build/index-operator \
+		docker/build/index-save \
+		docker/build/loadtest \
+		docker/build/manager-index \
+		docker/build/operator/helm \
+		docker/build/readreplica-rotate
+
 .PHONY: docker/name/org
 docker/name/org:
 	@echo "$(ORG)"

diff --git a/Makefile.d/tools.mk b/Makefile.d/tools.mk
@@ -89,7 +89,34 @@ textlint/ci/install:
 cspell/install: $(NPM_GLOBAL_PREFIX)/bin/cspell
 
 $(NPM_GLOBAL_PREFIX)/bin/cspell:
-	npm install -g cspell@latest
+	npm install -g cspell@latest \
+		@cspell/dict-cpp \
+		@cspell/dict-docker \
+		@cspell/dict-en_us \
+		@cspell/dict-fullstack \
+		@cspell/dict-git \
+		@cspell/dict-golang \
+		@cspell/dict-k8s \
+		@cspell/dict-makefile \
+		@cspell/dict-markdown \
+		@cspell/dict-npm \
+		@cspell/dict-public-licenses \
+		@cspell/dict-rust \
+		@cspell/dict-shell
+	cspell link add @cspell/dict-cpp
+	cspell link add @cspell/dict-docker
+	cspell link add @cspell/dict-en_us
+	cspell link add @cspell/dict-fullstack
+	cspell link add @cspell/dict-git
+	cspell link add @cspell/dict-golang
+	cspell link add @cspell/dict-k8s
+	cspell link add @cspell/dict-makefile
+	cspell link add @cspell/dict-markdown
+	cspell link add @cspell/dict-npm
+	cspell link add @cspell/dict-public-licenses
+	cspell link add @cspell/dict-rust
+	cspell link add @cspell/dict-shell
+
 
 .PHONY: buf/install
 buf/install: $(BINDIR)/buf