[bugfix] Vald gRPC Client and Pool logic makes huge backoff (#1953)

Signed-off-by: kpango <[email protected]>

CHANGELOG.md

@@ -65,11 +65,11 @@
 
 #### 🐛 Bugfix
 
-- [bugfix] add target length validation for each grpc client exection method [#1939](https://github.com/vdaas/vald/pull/1939)
+- [bugfix] add target length validation for each gRPC client exection method [#1939](https://github.com/vdaas/vald/pull/1939)
 
 #### ♻️ Refactor
 
-- update grpc status code for API docs [#1943](https://github.com/vdaas/vald/pull/1943)
+- update gRPC status code for API docs [#1943](https://github.com/vdaas/vald/pull/1943)
 - Refactor: Add t.Helper() on test helper function [#1935](https://github.com/vdaas/vald/pull/1935)
 - Fix syntax error on dump context workflow [#1936](https://github.com/vdaas/vald/pull/1936)
 - format codes [#1934](https://github.com/vdaas/vald/pull/1934)
@@ -2279,7 +2279,7 @@
 - :robot: Update license headers / Format Go codes and YAML files ([#1255](https://github.com/vdaas/vald/pull/1255))
 - :fire: Remove invalid initialization option ([#1252](https://github.com/vdaas/vald/pull/1252))
 - add single connection client for agent & vald ([#1254](https://github.com/vdaas/vald/pull/1254))
-- bugfix remove unneccessary error return & add grpc status code handling for backoff ([#1253](https://github.com/vdaas/vald/pull/1253))
+- bugfix remove unneccessary error return & add gRPC status code handling for backoff ([#1253](https://github.com/vdaas/vald/pull/1253))
 - bugfix agent vqueue & refactor tools/deps ([#1250](https://github.com/vdaas/vald/pull/1250))
 - Add test for pkg/agent/core/ngt/service/vqueue/option ([#1233](https://github.com/vdaas/vald/pull/1233))
 - Add config agent core ngt service kvs test ([#1223](https://github.com/vdaas/vald/pull/1223))
@@ -2381,7 +2381,7 @@
 
 #### Feature
 
-- add correct grpc status and error handling (#1224)
+- add correct gRPC status and error handling (#1224)
 - Add Cloud Storage mode of Agent Sidecar. (#519)
 - Add general info metrics package / Add NGT info metrics (#1195)
 - improve internal/info implementation (#1177)
@@ -3027,7 +3027,7 @@
 - change grpc bidi-stream error handling and change grpc API interface ([#928](https://github.com/vdaas/vald/pull/928))
 - fix unclosed string literal in Dockerfile's ARG MAINTAINER ([#923](https://github.com/vdaas/vald/pull/923))
 - Revise building workflow of ci and dev containers ([#922](https://github.com/vdaas/vald/pull/922))
-- bugfix add nil check for grpc connection pool objects in grpc/client.go ([#921](https://github.com/vdaas/vald/pull/921))
+- bugfix add nil check for gRPC connection pool objects in grpc/client.go ([#921](https://github.com/vdaas/vald/pull/921))
 - remove unneccessary pr-tag definition from chart ([#920](https://github.com/vdaas/vald/pull/920))
 - :pencil: Fix typo in gateway-vald configmap template ([#919](https://github.com/vdaas/vald/pull/919))
 - change docker base image PRIMARY_TAG name from nightly to latest ([#917](https://github.com/vdaas/vald/pull/917))
@@ -4803,7 +4803,7 @@
 ### Changes
 
 - async compressor
-- optimized gRPC pool connection
+- optimized gRPC connection pool
 - update helm chart API version
 - internal gRPC client for Vald
 - Cassandra NewConvictionPolicy

Makefile

@@ -319,6 +319,7 @@ clean:
 	rm -rf vendor
 	go clean -cache -modcache -testcache -i -r
 	mv ./apis/grpc/v1/vald/vald.go $(TEMP_DIR)/vald.go
+	mv ./apis/grpc/v1/agent/core/agent.go $(TEMP_DIR)/agent.go
 	mv ./apis/grpc/v1/payload/interface.go $(TEMP_DIR)/interface.go
 	rm -rf \
 		/go/pkg \
@@ -335,6 +336,8 @@ clean:
 		./go.mod
 	mkdir -p ./apis/grpc/v1/vald
 	mv $(TEMP_DIR)/vald.go ./apis/grpc/v1/vald/vald.go
+	mkdir -p ./apis/grpc/v1/agent/core
+	mv $(TEMP_DIR)/agent.go ./apis/grpc/v1/agent/core/agent.go
 	mkdir -p ./apis/grpc/v1/payload
 	mv $(TEMP_DIR)/interface.go ./apis/grpc/v1/payload/interface.go
 	cp ./hack/go.mod.default ./go.mod

Makefile.d/e2e.mk

@@ -19,6 +19,11 @@
 e2e:
 	$(call run-e2e-crud-test,-run TestE2EStandardCRUD)
 
+.PHONY: e2e/multi
+## run e2e multiple apis
+e2e/multi:
+	$(call run-e2e-multi-crud-test,-run TestE2EMultiAPIs)
+
 .PHONY: e2e/insert
 ## run insert e2e
 e2e/insert:

Makefile.d/functions.mk

@@ -104,3 +104,28 @@ define run-e2e-crud-test
 	    -portforward-pod-port=$(E2E_TARGET_PORT) \
 	    -namespace=$(E2E_TARGET_NAMESPACE)
 endef
+
+define run-e2e-multi-crud-test
+	go test \
+	    -race \
+	    -mod=readonly \
+	    $1 \
+            -v $(ROOTDIR)/tests/e2e/multiapis/multiapis_test.go \
+	    -tags "e2e" \
+	    -timeout $(E2E_TIMEOUT) \
+	    -host=$(E2E_BIND_HOST) \
+	    -port=$(E2E_BIND_PORT) \
+	    -dataset=$(ROOTDIR)/hack/benchmark/assets/dataset/$(E2E_DATASET_NAME).hdf5 \
+	    -insert-num=$(E2E_INSERT_COUNT) \
+	    -search-num=$(E2E_SEARCH_COUNT) \
+	    -search-by-id-num=$(E2E_SEARCH_BY_ID_COUNT) \
+	    -get-object-num=$(E2E_GET_OBJECT_COUNT) \
+	    -update-num=$(E2E_UPDATE_COUNT) \
+	    -upsert-num=$(E2E_UPSERT_COUNT) \
+	    -remove-num=$(E2E_REMOVE_COUNT) \
+	    -wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \
+	    -portforward=$(E2E_PORTFORWARD_ENABLED) \
+	    -portforward-pod-name=$(E2E_TARGET_POD_NAME) \
+	    -portforward-pod-port=$(E2E_TARGET_PORT) \
+	    -namespace=$(E2E_TARGET_NAMESPACE)
+endef

internal/net/grpc/pool/pool_test.go → apis/grpc/v1/agent/core/agent.go

@@ -14,15 +14,12 @@
 // limitations under the License.
 //
 
-// Package pool provides grpc connection pool client
-package pool
+// Package core provides core agent server interface
+package core
 
-import (
-	"testing"
-
-	"github.com/vdaas/vald/internal/test/goleak"
+const (
+	CreateIndexRPCName        = "CreateIndex"
+	SaveIndexRPCName          = "SaveIndex"
+	CreateAndSaveIndexRPCName = "CreateAndSaveIndex"
+	IndexInfoRPCName          = "IndexInfo"
 )
-
-func TestMain(m *testing.M) {
-	goleak.VerifyTestMain(m)
-}

apis/swagger/v1/agent/core/apis/proto/v1/agent/core/agent.swagger.json

@@ -150,7 +150,7 @@
           "description": "Must be a valid serialized protocol buffer of the above specified type."
         }
       },
-      "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n    Foo foo = ...;\n    Any any;\n    any.PackFrom(foo);\n    ...\n    if (any.UnpackTo(\u0026foo)) {\n      ...\n    }\n\nExample 2: Pack and unpack a message in Java.\n\n    Foo foo = ...;\n    Any any = Any.pack(foo);\n    ...\n    if (any.is(Foo.class)) {\n      foo = any.unpack(Foo.class);\n    }\n\nExample 3: Pack and unpack a message in Python.\n\n    foo = Foo(...)\n    any = Any()\n    any.Pack(foo)\n    ...\n    if any.Is(Foo.DESCRIPTOR):\n      any.Unpack(foo)\n      ...\n\nExample 4: Pack and unpack a message in Go\n\n     foo := \u0026pb.Foo{...}\n     any, err := anypb.New(foo)\n     if err != nil {\n       ...\n     }\n     ...\n     foo := \u0026pb.Foo{}\n     if err := any.UnmarshalTo(foo); err != nil {\n       ...\n     }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n    package google.profile;\n    message Person {\n      string first_name = 1;\n      string last_name = 2;\n    }\n\n    {\n      \"@type\": \"type.googleapis.com/google.profile.Person\",\n      \"firstName\": \u003cstring\u003e,\n      \"lastName\": \u003cstring\u003e\n    }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n    {\n      \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n      \"value\": \"1.212s\"\n    }"
+      "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n    Foo foo = ...;\n    Any any;\n    any.PackFrom(foo);\n    ...\n    if (any.UnpackTo(\u0026foo)) {\n      ...\n    }\n\nExample 2: Pack and unpack a message in Java.\n\n    Foo foo = ...;\n    Any any = Any.pack(foo);\n    ...\n    if (any.is(Foo.class)) {\n      foo = any.unpack(Foo.class);\n    }\n    // or ...\n    if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n      foo = any.unpack(Foo.getDefaultInstance());\n    }\n\nExample 3: Pack and unpack a message in Python.\n\n    foo = Foo(...)\n    any = Any()\n    any.Pack(foo)\n    ...\n    if any.Is(Foo.DESCRIPTOR):\n      any.Unpack(foo)\n      ...\n\nExample 4: Pack and unpack a message in Go\n\n     foo := \u0026pb.Foo{...}\n     any, err := anypb.New(foo)\n     if err != nil {\n       ...\n     }\n     ...\n     foo := \u0026pb.Foo{}\n     if err := any.UnmarshalTo(foo); err != nil {\n       ...\n     }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n    package google.profile;\n    message Person {\n      string first_name = 1;\n      string last_name = 2;\n    }\n\n    {\n      \"@type\": \"type.googleapis.com/google.profile.Person\",\n      \"firstName\": \u003cstring\u003e,\n      \"lastName\": \u003cstring\u003e\n    }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n    {\n      \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n      \"value\": \"1.212s\"\n    }"
     },
     "runtimeError": {
       "type": "object",

apis/swagger/v1/discoverer/apis/proto/v1/discoverer/discoverer.swagger.json

@@ -216,7 +216,7 @@
           "description": "Must be a valid serialized protocol buffer of the above specified type."
         }
       },
-      "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n    Foo foo = ...;\n    Any any;\n    any.PackFrom(foo);\n    ...\n    if (any.UnpackTo(\u0026foo)) {\n      ...\n    }\n\nExample 2: Pack and unpack a message in Java.\n\n    Foo foo = ...;\n    Any any = Any.pack(foo);\n    ...\n    if (any.is(Foo.class)) {\n      foo = any.unpack(Foo.class);\n    }\n\nExample 3: Pack and unpack a message in Python.\n\n    foo = Foo(...)\n    any = Any()\n    any.Pack(foo)\n    ...\n    if any.Is(Foo.DESCRIPTOR):\n      any.Unpack(foo)\n      ...\n\nExample 4: Pack and unpack a message in Go\n\n     foo := \u0026pb.Foo{...}\n     any, err := anypb.New(foo)\n     if err != nil {\n       ...\n     }\n     ...\n     foo := \u0026pb.Foo{}\n     if err := any.UnmarshalTo(foo); err != nil {\n       ...\n     }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n    package google.profile;\n    message Person {\n      string first_name = 1;\n      string last_name = 2;\n    }\n\n    {\n      \"@type\": \"type.googleapis.com/google.profile.Person\",\n      \"firstName\": \u003cstring\u003e,\n      \"lastName\": \u003cstring\u003e\n    }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n    {\n      \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n      \"value\": \"1.212s\"\n    }"
+      "description": "`Any` contains an arbitrary serialized protocol buffer message along with a\nURL that describes the type of the serialized message.\n\nProtobuf library provides support to pack/unpack Any values in the form\nof utility functions or additional generated methods of the Any type.\n\nExample 1: Pack and unpack a message in C++.\n\n    Foo foo = ...;\n    Any any;\n    any.PackFrom(foo);\n    ...\n    if (any.UnpackTo(\u0026foo)) {\n      ...\n    }\n\nExample 2: Pack and unpack a message in Java.\n\n    Foo foo = ...;\n    Any any = Any.pack(foo);\n    ...\n    if (any.is(Foo.class)) {\n      foo = any.unpack(Foo.class);\n    }\n    // or ...\n    if (any.isSameTypeAs(Foo.getDefaultInstance())) {\n      foo = any.unpack(Foo.getDefaultInstance());\n    }\n\nExample 3: Pack and unpack a message in Python.\n\n    foo = Foo(...)\n    any = Any()\n    any.Pack(foo)\n    ...\n    if any.Is(Foo.DESCRIPTOR):\n      any.Unpack(foo)\n      ...\n\nExample 4: Pack and unpack a message in Go\n\n     foo := \u0026pb.Foo{...}\n     any, err := anypb.New(foo)\n     if err != nil {\n       ...\n     }\n     ...\n     foo := \u0026pb.Foo{}\n     if err := any.UnmarshalTo(foo); err != nil {\n       ...\n     }\n\nThe pack methods provided by protobuf library will by default use\n'type.googleapis.com/full.type.name' as the type URL and the unpack\nmethods only use the fully qualified type name after the last '/'\nin the type URL, for example \"foo.bar.com/x/y.z\" will yield type\nname \"y.z\".\n\nJSON\n\nThe JSON representation of an `Any` value uses the regular\nrepresentation of the deserialized, embedded message, with an\nadditional field `@type` which contains the type URL. Example:\n\n    package google.profile;\n    message Person {\n      string first_name = 1;\n      string last_name = 2;\n    }\n\n    {\n      \"@type\": \"type.googleapis.com/google.profile.Person\",\n      \"firstName\": \u003cstring\u003e,\n      \"lastName\": \u003cstring\u003e\n    }\n\nIf the embedded message type is well-known and has a custom JSON\nrepresentation, that representation will be embedded adding a field\n`value` which holds the custom JSON in addition to the `@type`\nfield. Example (for message [google.protobuf.Duration][]):\n\n    {\n      \"@type\": \"type.googleapis.com/google.protobuf.Duration\",\n      \"value\": \"1.212s\"\n    }"
     },
     "runtimeError": {
       "type": "object",