Add e2e test for maxDimensionTest

.github/helm/values/values-max-dim.yaml

@@ -0,0 +1,71 @@
+#
+# Copyright (C) 2019-2022 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  logging:
+    level: info
+
+gateway:
+  lb:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 1
+
+agent:
+  minReplicas: 1
+  maxReplicas: 1
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  ngt:
+    auto_index_duration_limit: 1m
+    auto_index_check_duration: 40s
+    auto_index_length: 100
+    dimension: 2
+
+discoverer:
+  minReplicas: 1
+  maxReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+
+manager:
+  index:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 40s
+      auto_index_check_duration: 20s
+      auto_index_length: 100

.github/workflows/e2e-max-dim.yml

@@ -0,0 +1,164 @@
+#
+# Copyright (C) 2019-2022 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Run E2E Max Dimension test"
+on:
+  push:
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+  pull_request:
+    types:
+      - "labeled"
+
+jobs:
+  e2e-max-dimension-insert:
+    name: "E2E test (Max Dimension Insert: skip strict exist check)"
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Wait for Docker images
+        if: startsWith( github.ref, 'refs/tags/')
+        id: wait_for_docker_images
+        uses: ./.github/actions/wait-for-docker-image
+      - name: Get PR number
+        id: get_pr_number
+        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
+        run: |
+          pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+          echo "::set-output name=PR_NUM::${pr_num}"
+      - name: Specify container versions
+        if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy'
+        id: specify_container_versions
+        uses: ./.github/actions/detect-docker-image-tags
+        with:
+          tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }}
+      - name: Fetch Helm version
+        run: |
+          HELM_VERSION=`make version/helm`
+          echo "::set-output name=helm::${HELM_VERSION}"
+        id: version
+      - uses: rinx/[email protected]
+        with:
+          version: latest
+          name: vald
+          agents: 1
+      - name: check k3d
+        run: |
+          kubectl cluster-info
+      - uses: azure/setup-helm@v1
+        with:
+          version: ${{ steps.version.outputs.helm }}
+      - name: Helm version
+        run: |
+          helm version
+      - name: deploy Vald
+        id: deploy_vald
+        uses: ./.github/actions/e2e-deploy-vald
+        with:
+          helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }}
+          values: .github/helm/values/values-max-dim.yaml
+          wait_for_selector: app=vald-lb-gateway
+      - name: Fetch golang version
+        run: |
+          GO_VERSION=`make version/go`
+          echo "::set-output name=version::${GO_VERSION}"
+        id: golang_version
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.golang_version.outputs.version }}
+      - name: run E2E MaxDimension
+        run: |
+          go version
+          go get github.com/vdaas/vald-client-go/v1/payload
+          go get github.com/vdaas/vald-client-go/v1/vald
+          BIT=1
+          RELEASE=`helm list | grep vald | awk '{print $1}'`
+          while [ $BIT -ne 32 ]; do
+            echo "Start: Bit=${BIT}"
+            FILE="tmp.log"
+            touch ${FILE}
+            FILEPATH=`readlink -f ${FILE}`
+            DIM=$((1<<$BIT))
+            if [ $BIT = "32" ]; then
+              DIM=$(($DIM-1))
+            fi
+            helm upgrade ${RELEASE} charts/vald \
+              --values ${VALUES} \
+              --set defaults.image.tag=nightly \
+              --set agent.ngt.dimension=${DIM}
+            sleep 3
+            kubectl rollout restart statefulset vald-agent-ngt
+            sleep 30
+            kubectl wait --for=condition=Ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
+            POD_NAME=`kubectl get pods | grep vald-lb-gateway | awk '{print $1}'`
+            go test \
+              -race \
+              -v tests/e2e/performance/max_vector_dim_test.go \
+              -tags "e2e" \
+              -file ${FILEPATH} \
+              -timeout 30m \
+              -bit=${BIT} \
+              -host=localhost \
+              -port=8081 \
+              -portforward \
+              -portforward-pod-name=${POD_NAME} \
+              -kubeconfig=${KUBECONFIG} \
+              -namespace=default
+            CODE=`sed -n 1P ${FILEPATH}`
+            if [ ${CODE} = "ResourceExhausted" ]; then
+              echo "Finish: Bit=${BIT} with ${CODE}"
+              BIT=$(($BIT-1))
+              rm ${FILEPATH}
+              break;
+            fi
+            if [ ${CODE} != "OK" ]; then
+              echo "Finish: Bit=${BIT} with Error: ${CODE}"
+              rm ${FILEPATH}
+              break;
+            fi
+            echo "Finish: Bit=${BIT}"
+            BIT=$(($BIT+1))
+            rm ${FILEPATH}
+          done
+          echo "::set-output name=MAX_BIT::${BIT}"
+          echo "MAX_BIT=${BIT}"
+        env:
+          WAIT_FOR_SELECTOR: app=vald-agent-ngt
+          WAIT_FOR_TIMEOUT: 29m
+          VALUES: .github/helm/values/values-max-dim.yaml
+        timeout-minutes: 60
+  slack-notification:
+    name: "Slack notification"
+    needs:
+      - e2e-max-dimension-insert
+    runs-on: ubuntu-latest
+    if: startsWith( github.ref, 'refs/tags/')
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: 8398a7/action-slack@v3
+        with:
+          author_name: "E2E max dim test"
+          status: ${{ env.WORKFLOW_CONCLUSION }}
+          only_mention_fail: channel
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}

internal/core/algorithm/algorithm.go

@@ -21,7 +21,7 @@ import "math"
 
 const (
 	// MaximumVectorDimensionSize is maximum value of vector dimension.
-	MaximumVectorDimensionSize = math.MaxInt64
+	MaximumVectorDimensionSize = math.MaxUint32
 	// MinimumVectorDimensionSize is minimum value of vector dimension.
 	MinimumVectorDimensionSize = 2
 )

internal/core/algorithm/ngt/ngt.go

@@ -152,13 +152,6 @@ const (
 	// -------------------------------------------------------------.
 	ErrorCode = C._Bool(false)
 	// -------------------------------------------------------------.
-
-	// -------------------------------------------------------------
-	// dimension constraints
-	// -------------------------------------------------------------.
-	VectorDimensionSizeLimit = 1<<32 - 1
-	minimumDimensionSize     = algorithm.MinimumVectorDimensionSize
-	// -------------------------------------------------------------.
 )
 
 func (o objectType) String() string {
@@ -685,8 +678,8 @@ func (n *ngt) GetVector(id uint) ([]float32, error) {
 		if results == nil {
 			return nil, n.newGoError(ebuf)
 		}
-		ret = (*[VectorDimensionSizeLimit]float32)(unsafe.Pointer(results))[:dimension:dimension]
-		// for _, elem := range (*[VectorDimensionSizeLimit]C.float)(unsafe.Pointer(results))[:dimension:dimension]{
+		ret = (*[algorithm.MaximumVectorDimensionSize]float32)(unsafe.Pointer(results))[:dimension:dimension]
+		// for _, elem := range (*[algorithm.MaximumVectorDimensionSize]C.float)(unsafe.Pointer(results))[:dimension:dimension]{
 		// 	ret = append(ret, float32(elem))
 		// }
 	case Uint8:
@@ -697,7 +690,7 @@ func (n *ngt) GetVector(id uint) ([]float32, error) {
 			return nil, n.newGoError(ebuf)
 		}
 		ret = make([]float32, 0, dimension)
-		for _, elem := range (*[VectorDimensionSizeLimit]C.uint8_t)(unsafe.Pointer(results))[:dimension:dimension] {
+		for _, elem := range (*[algorithm.MaximumVectorDimensionSize]C.uint8_t)(unsafe.Pointer(results))[:dimension:dimension] {
 			ret = append(ret, float32(elem))
 		}
 	default:

internal/core/algorithm/ngt/ngt_test.go

@@ -26,6 +26,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/vdaas/vald/internal/core/algorithm"
 	"github.com/vdaas/vald/internal/errors"
 	"github.com/vdaas/vald/internal/file"
 	"github.com/vdaas/vald/internal/log"
@@ -183,7 +184,7 @@ func TestNew(t *testing.T) {
 				},
 			},
 			want: want{
-				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, VectorDimensionSizeLimit)),
+				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, algorithm.MaximumVectorDimensionSize)),
 			},
 		},
 	}
@@ -761,7 +762,7 @@ func Test_gen(t *testing.T) {
 				},
 			},
 			want: want{
-				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, VectorDimensionSizeLimit)),
+				err: errors.NewErrCriticalOption("dimension", 1, errors.ErrInvalidDimensionSize(1, algorithm.MaximumVectorDimensionSize)),
 			},
 		},
 	}

internal/core/algorithm/ngt/option.go

@@ -27,6 +27,7 @@ import (
 	"strconv"
 
 	"github.com/kpango/fastime"
+	"github.com/vdaas/vald/internal/core/algorithm"
 	"github.com/vdaas/vald/internal/errors"
 	"github.com/vdaas/vald/internal/strings"
 )
@@ -41,7 +42,7 @@ var (
 
 	defaultOptions = []Option{
 		WithIndexPath("/tmp/ngt-" + strconv.FormatInt(fastime.UnixNanoNow(), 10)),
-		WithDimension(minimumDimensionSize),
+		WithDimension(algorithm.MinimumVectorDimensionSize),
 		WithDefaultRadius(DefaultRadius),
 		WithDefaultEpsilon(DefaultEpsilon),
 		WithDefaultPoolSize(DefaultPoolSize),
@@ -86,8 +87,8 @@ func WithBulkInsertChunkSize(size int) Option {
 // WithDimension represents the option to set the dimension for NGT.
 func WithDimension(size int) Option {
 	return func(n *ngt) error {
-		if size > VectorDimensionSizeLimit || size < minimumDimensionSize {
-			err := errors.ErrInvalidDimensionSize(size, VectorDimensionSizeLimit)
+		if size > algorithm.MaximumVectorDimensionSize || size < algorithm.MinimumVectorDimensionSize {
+			err := errors.ErrInvalidDimensionSize(size, algorithm.MaximumVectorDimensionSize)
 			return errors.NewErrCriticalOption("dimension", size, err)
 		}