Bug fix for gateway

.github/helm/values/values-scylla.yaml

@@ -62,8 +62,8 @@ agent:
       cpu: 100m
       memory: 50Mi
   ngt:
-    auto_index_duration_limit: 3m
-    auto_index_check_duration: 1m
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
     auto_index_length: 1000
     dimension: 784
 
@@ -122,6 +122,10 @@ manager:
       requests:
         cpu: 100m
         memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000
 
 meta:
   minReplicas: 3
@@ -150,3 +154,17 @@ meta:
       hosts:
         - vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local
       consistency: one
+
+initializer:
+  cassandra:
+    enabled: true
+    name: scylla-init
+    env:
+      - name: CASSANDRA_HOST
+        value: vald-scylla-cluster-dc0-rack0-0.scylla.svc.cluster.local
+    configmap:
+      enabled: true
+      name: scylla-initdb
+      user: ""
+    secret:
+      enabled: false

.github/workflows/e2e-deploy.yml

@@ -110,8 +110,8 @@ jobs:
         run: |
           sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
 
-          make k8s/external/mysql/deploy
-          make k8s/external/redis/deploy
+          make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/mysql/deploy
+          make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/redis/deploy
 
           helm repo add vald https://vald.vdaas.org/charts
           tag=$(cat versions/VALD_VERSION)
@@ -143,7 +143,7 @@ jobs:
             -get-object-num=5 \
             -update-num=2 \
             -remove-num=2 \
-            -wait-after-insert=2m \
+            -wait-after-insert=3m \
             -portforward \
             -portforward-ns=default \
             -portforward-pod-name=${podname} \
@@ -233,7 +233,7 @@ jobs:
           helm version
       - name: deploy vald
         run: |
-          make k8s/external/scylla/deploy
+          make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/scylla/deploy
 
           helm repo add vald https://vald.vdaas.org/charts
           tag=$(cat versions/VALD_VERSION)
@@ -265,7 +265,7 @@ jobs:
             -get-object-num=10 \
             -update-num=3 \
             -remove-num=2 \
-            -wait-after-insert=2m \
+            -wait-after-insert=3m \
             -portforward \
             -portforward-ns=default \
             -portforward-pod-name=${podname} \
@@ -341,7 +341,7 @@ jobs:
           helm version
       - name: deploy vald
         run: |
-          make k8s/external/minio/deploy
+          make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy
 
           helm repo add vald https://vald.vdaas.org/charts
           tag=$(cat versions/VALD_VERSION)

Makefile

@@ -223,14 +223,31 @@ DISTROLESS_IMAGE      ?= gcr.io/distroless/static
 DISTROLESS_IMAGE_TAG  ?= nonroot
 UPX_OPTIONS           ?= -9
 
-K8S_EXTERNAL_SCYLLA_MANIFEST ?= k8s/external/scylla/scyllacluster.yaml
+K8S_EXTERNAL_SCYLLA_MANIFEST        ?= k8s/external/scylla/scyllacluster.yaml
+K8S_SLEEP_DURATION_FOR_WAIT_COMMAND ?= 5
 
 K8S_KUBECTL_VERSION ?= $(eval K8S_KUBECTL_VERSION := $(shell kubectl version --short))$(K8S_KUBECTL_VERSION)
 K8S_SERVER_VERSION ?= $(eval K8S_SERVER_VERSION := $(shell echo "$(K8S_KUBECTL_VERSION)" | sed -e "s/.*Server.*\(v[0-9]\.[0-9]*\)\..*/\1/g"))$(K8S_SERVER_VERSION)
 
 COMMA := ,
 SHELL = bash
 
+E2E_BIND_HOST                      ?= 127.0.0.1
+E2E_BIND_PORT                      ?= 8082
+E2E_TIMEOUT                        ?= 15m
+E2E_DATASET_NAME                   ?= fashion-mnist-784-euclidean
+E2E_INSERT_COUNT                   ?= 1000
+E2E_SEARCH_COUNT                   ?= 1000
+E2E_SEARCH_BY_ID_COUNT             ?= 10
+E2E_GET_OBJECT_COUNT               ?= 10
+E2E_UPDATE_COUNT                   ?= 3
+E2E_REMOVE_COUNT                   ?= 3
+E2E_WAIT_FOR_CREATE_INDEX_DURATION ?= 3m
+E2E_TARGET_NAME                    ?= vald-meta-gateway
+E2E_TARGET_POD_NAME                ?= $(eval E2E_TARGET_POD_NAME := $(shell kubectl get pods --selector=app=$(E2E_TARGET_NAME) | tail -1 | cut -f1 -d " "))$(E2E_TARGET_POD_NAME)
+E2E_TARGET_NAMESPACE               ?= default
+E2E_TARGET_PORT                    ?= 8081
+
 include Makefile.d/functions.mk
 
 .PHONY: maintainer
@@ -468,3 +485,4 @@ include Makefile.d/kind.mk
 include Makefile.d/client.mk
 include Makefile.d/ml.mk
 include Makefile.d/test.mk
+include Makefile.d/e2e.mk

Makefile.d/e2e.mk

@@ -0,0 +1,41 @@
+#
+# Copyright (C) 2019-2021 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.PHONY: e2e
+## run e2e
+e2e:
+	$(call run-e2e-test)
+
+.PHONY: e2e/insert
+## run insert e2e
+e2e/insert:
+	$(call run-e2e-test,-run TestE2EInsert)
+
+
+.PHONY: e2e/update
+## run update e2e
+e2e/update:
+	$(call run-e2e-test,-run TestE2EUpdate)
+
+.PHONY: e2e/search
+## run search e2e
+e2e/search:
+	$(call run-e2e-test,-run TestE2ESearch)
+
+.PHONY: e2e/remove
+## run remove e2e
+e2e/remove:
+	$(call run-e2e-test,-run TestE2ERemove)

Makefile.d/functions.mk

@@ -65,3 +65,26 @@ define telepresence
 	    ## https://github.com/telepresenceio/telepresence/commit/bb7473fbf19ed4f61796a5e32747e23de6ab03da
 	    ## --deployment-type "$(SWAP_DEPLOYMENT_TYPE)"
 endef
+
+
+define run-e2e-test
+	go test \
+	    $1 \
+	    -v $(ROOTDIR)/tests/e2e/crud_test.go \
+	    -tags "e2e" \
+	    -timeout $(E2E_TIMEOUT) \
+	    -host=$(E2E_BIND_HOST) \
+	    -port=$(E2E_BIND_PORT) \
+	    -dataset=$(ROOTDIR)/hack/benchmark/assets/dataset/$(E2E_DATASET_NAME).hdf5 \
+	    -insert-num=$(E2E_INSERT_COUNT) \
+	    -search-num=$(E2E_SEARCH_COUNT) \
+	    -search-by-id-num=$(E2E_SEARCH_BY_ID_COUNT) \
+	    -get-object-num=$(E2E_GET_OBJECT_COUNT) \
+	    -update-num=$(E2E_UPDATE_COUNT) \
+	    -remove-num=$(E2E_REMOVE_COUNT) \
+	    -wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \
+	    -portforward-ns=$(E2E_TARGET_NAMESPACE) \
+	    -portforward-pod-name=$(E2E_TARGET_POD_NAME) \
+	    -portforward-pod-port=$(E2E_TARGET_PORT) \
+	    -portforward
+endef

Makefile.d/k8s.mk

@@ -89,6 +89,7 @@ k8s/vald/deploy: \
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/backup
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/meta
 	rm -rf $(TEMP_DIR)
+	kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n"
 
 .PHONY: k8s/vald/delete
 ## delete vald sample cluster from k8s
@@ -129,6 +130,7 @@ k8s/vald/deploy/cassandra: \
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/backup
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/meta
 	rm -rf $(TEMP_DIR)
+	kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n"
 
 
 .PHONY: k8s/vald/delete/cassandra
@@ -175,6 +177,7 @@ k8s/vald/deploy/scylla: \
 	kubectl apply -f $(TEMP_DIR)/vald/templates/meta
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/meta
 	rm -rf $(TEMP_DIR)
+	kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n"
 
 .PHONY: k8s/vald/delete/scylla
 ## delete vald sample cluster with scylla to k8s
@@ -203,6 +206,7 @@ k8s/vald/delete/scylla: \
 k8s/external/mysql/deploy:
 	kubectl apply -f k8s/jobs/db/initialize/mysql/configmap.yaml
 	kubectl apply -f k8s/external/mysql
+	sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND)
 	kubectl wait --for=condition=ready pod -l app=mysql --timeout=600s
 
 .PHONY: k8s/external/mysql/delete
@@ -222,6 +226,7 @@ k8s/external/mysql/initialize:
 ## deploy redis to k8s
 k8s/external/redis/deploy:
 	kubectl apply -f k8s/external/redis
+	sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND)
 	kubectl wait --for=condition=ready pod -l app=redis --timeout=600s
 
 .PHONY: k8s/external/redis/delete
@@ -240,7 +245,7 @@ k8s/external/redis/initialize:
 ## deploy cassandra to k8s
 k8s/external/cassandra/deploy:
 	kubectl apply -f https://raw.githubusercontent.com/datastax/cass-operator/master/docs/user/cass-operator-manifests-$(K8S_SERVER_VERSION).yaml
-	sleep 2
+	sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND)
 	kubectl apply -n cass-operator -f k8s/jobs/db/initialize/cassandra/secret.yaml
 	kubectl wait -n cass-operator --for=condition=ready pod -l name=cass-operator --timeout=600s
 	kubectl apply -f k8s/external/cassandra
@@ -266,12 +271,13 @@ k8s/external/cassandra/initialize:
 k8s/external/scylla/deploy: \
 	k8s/external/cert-manager/deploy
 	kubectl apply -f https://raw.githubusercontent.com/scylladb/scylla-operator/master/examples/common/operator.yaml
-	sleep 2
+	sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND)
 	kubectl wait -n scylla-operator-system --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=scylla-operator-controller-manager-0 --timeout=600s
 	kubectl -n scylla-operator-system get pod
 	kubectl apply -f $(K8S_EXTERNAL_SCYLLA_MANIFEST)
 	kubectl -n scylla get ScyllaCluster
 	kubectl -n scylla get pods
+	sleep 1
 	kubectl wait -n scylla --for=condition=ready pod -l statefulset.kubernetes.io/pod-name=vald-scylla-cluster-dc0-rack0-0 --timeout=600s
 	kubectl -n scylla get ScyllaCluster
 	kubectl -n scylla get pods
@@ -287,7 +293,7 @@ k8s/external/scylla/delete: \
 ## deploy cert-manager
 k8s/external/cert-manager/deploy:
 	kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml
-	sleep 2
+	sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND)
 	kubectl wait -n cert-manager --for=condition=ready pod -l app=cert-manager --timeout=60s
 	kubectl wait -n cert-manager --for=condition=ready pod -l app=cainjector --timeout=60s
 	kubectl wait -n cert-manager --for=condition=ready pod -l app=webhook --timeout=60s
@@ -303,6 +309,7 @@ k8s/external/cert-manager/delete:
 ## deploy minio
 k8s/external/minio/deploy:
 	kubectl apply -f k8s/external/minio
+	sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND)
 	kubectl wait --for=condition=ready pod -l app=minio --timeout=600s
 
 .PHONY: k8s/external/minio/delete
@@ -314,6 +321,7 @@ k8s/external/minio/delete:
 ## deploy metrics-serrver
 k8s/metrics/metrics-server/deploy:
 	kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
+	sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND)
 	kubectl wait -n kube-system --for=condition=ready pod -l k8s-app=metrics-server --timeout=600s
 
 .PHONY: k8s/metrics/metrics-server/delete