add inner product distance type for ngt

Signed-off-by: kpango <[email protected]>

.gitfiles

@@ -1048,6 +1048,7 @@ internal/k8s/service/option.go
 internal/k8s/service/option_test.go
 internal/k8s/service/service.go
 internal/k8s/service/service_test.go
+internal/k8s/vald/annotations.go
 internal/k8s/vald/benchmark/api/v1/info.go
 internal/k8s/vald/benchmark/api/v1/job_types.go
 internal/k8s/vald/benchmark/api/v1/scenario_types.go
@@ -1214,6 +1215,8 @@ internal/observability/metrics/runtime/cgo/cgo.go
 internal/observability/metrics/runtime/cgo/cgo_test.go
 internal/observability/metrics/runtime/goroutine/goroutine.go
 internal/observability/metrics/runtime/goroutine/goroutine_test.go
+internal/observability/metrics/tools/benchmark/benchmark.go
+internal/observability/metrics/tools/benchmark/benchmark_test.go
 internal/observability/metrics/version/version.go
 internal/observability/metrics/version/version_test.go
 internal/observability/observability.go
@@ -1436,6 +1439,7 @@ k8s/metrics/grafana/dashboards/05-vald-index-manager.yaml
 k8s/metrics/grafana/dashboards/07-vald-helm-operator.yaml
 k8s/metrics/grafana/dashboards/08-vald-lb-gateway.yaml
 k8s/metrics/grafana/dashboards/09-vald-index-correction.yaml
+k8s/metrics/grafana/dashboards/10-vald-benchmark-operator.yaml
 k8s/metrics/grafana/dashboards/99-vald-agent-memory.yaml
 k8s/metrics/grafana/deployment.yaml
 k8s/metrics/grafana/svc.yaml

.yamlfmt

@@ -0,0 +1,18 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+formatter:
+  type: basic
+  indentation: 2

Makefile

@@ -569,13 +569,12 @@ version/telepresence:
 ## install NGT
 ngt/install: /usr/local/include/NGT/Capi.h
 /usr/local/include/NGT/Capi.h:
-	curl -LO https://github.com/yahoojapan/NGT/archive/v$(NGT_VERSION).tar.gz
-	tar zxf v$(NGT_VERSION).tar.gz -C $(TEMP_DIR)/
+	git clone --depth 1 --branch v$(NGT_VERSION) https://github.com/yahoojapan/NGT $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	cd $(TEMP_DIR)/NGT-$(NGT_VERSION) && \
 		cmake -DCMAKE_C_FLAGS="$(CFLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" .
 	make -j -C $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	make install -C $(TEMP_DIR)/NGT-$(NGT_VERSION)
-	rm -rf v$(NGT_VERSION).tar.gz
+	cd $(ROOTDIR)
 	rm -rf $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	ldconfig
 

charts/vald-benchmark-operator/values.schema.json

@@ -8,6 +8,11 @@
       "type": "object",
       "description": "deployment annotations"
     },
+    "env": {
+      "type": "array",
+      "description": "environment variables",
+      "items": { "type": "object" }
+    },
     "image": {
       "type": "object",
       "properties": {

charts/vald-helm-operator/crds/valdrelease.yaml

@@ -263,6 +263,10 @@ spec:
                             - normalizedangle
                             - normcos
                             - normalizedcosine
+                            - dotproduct
+                            - innerproduct
+                            - dp
+                            - ip
                         enable_copy_on_write:
                           type: boolean
                         enable_export_index_info_to_k8s:

charts/vald/values.schema.json

@@ -231,7 +231,7 @@
             },
             "distance_type": {
               "type": "string",
-              "description": "distance type. it should be `l1`, `l2`, `angle`, `hamming`, `cosine`,`poincare`, `lorentz`, `jaccard`, `sparsejaccard`, `normalizedangle` or `normalizedcosine`. for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/algorithm/ngt#pkg-constants",
+              "description": "distance type. it should be `l1`, `l2`, `angle`, `hamming`, `cosine`,`poincare`, `lorentz`, `jaccard`, `sparsejaccard`, `normalizedangle` or `normalizedcosine` or `innerproduct`. for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/algorithm/ngt#pkg-constants",
               "enum": [
                 "l1",
                 "l2",
@@ -254,7 +254,11 @@
                 "normang",
                 "normalizedangle",
                 "normcos",
-                "normalizedcosine"
+                "normalizedcosine",
+                "dotproduct",
+                "innerproduct",
+                "dp",
+                "ip"
               ]
             },
             "enable_copy_on_write": {

charts/vald/values.yaml

@@ -2092,9 +2092,9 @@ agent:
     # @schema {"name": "agent.ngt.bulk_insert_chunk_size", "type": "integer"}
     # agent.ngt.bulk_insert_chunk_size -- bulk insert chunk size
     bulk_insert_chunk_size: 10
-    # @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "ang", "angle", "ham", "hamming", "cos", "cosine", "poincare", "poinc", "lorentz", "loren", "jac", "jaccard", "spjac", "sparsejaccard", "norml2", "normalizedl2", "normang", "normalizedangle", "normcos", "normalizedcosine"]}
+    # @schema {"name": "agent.ngt.distance_type", "type": "string", "enum": ["l1", "l2", "ang", "angle", "ham", "hamming", "cos", "cosine", "poincare", "poinc", "lorentz", "loren", "jac", "jaccard", "spjac", "sparsejaccard", "norml2", "normalizedl2", "normang", "normalizedangle", "normcos", "normalizedcosine", "dotproduct", "innerproduct", "dp", "ip"]}
     # agent.ngt.distance_type -- distance type.
-    # it should be `l1`, `l2`, `angle`, `hamming`, `cosine`,`poincare`, `lorentz`, `jaccard`, `sparsejaccard`, `normalizedangle` or `normalizedcosine`.
+    # it should be `l1`, `l2`, `angle`, `hamming`, `cosine`,`poincare`, `lorentz`, `jaccard`, `sparsejaccard`, `normalizedangle` or `normalizedcosine` or `innerproduct`.
     # for further details about NGT libraries supported distance is https://github.com/yahoojapan/NGT/wiki/Command-Quick-Reference
     # and vald agent's supported NGT distance type is https://pkg.go.dev/github.com/vdaas/vald/internal/core/algorithm/ngt#pkg-constants
     distance_type: l2

example/client/go.mod

@@ -11,9 +11,9 @@ replace (
 	golang.org/x/crypto => golang.org/x/crypto v0.21.0
 	golang.org/x/net => golang.org/x/net v0.22.0
 	golang.org/x/text => golang.org/x/text v0.14.0
-	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7
-	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7
-	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7
+	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240314234333-6e1732d8331c
+	google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c
+	google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c
 	google.golang.org/grpc => google.golang.org/grpc v1.62.1
 	google.golang.org/protobuf => google.golang.org/protobuf v1.33.0
 	gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0

example/client/go.sum

@@ -34,10 +34,10 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM=
 gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I=
-google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E=
-google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc=
+google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
 google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=