add broken index backup

.devcontainer/devcontainer.json

@@ -10,5 +10,6 @@
       "moby": "true"
     }
   },
-  "postCreateCommand": "go version"
+  "postCreateCommand": "go version",
+  "postAttachCommand": "sudo ln -s $(pwd)/cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml"
 }

.gitignore

@@ -51,4 +51,7 @@ hack/go.mod.default3
 # for nvim
 .nvimlog
 
+# for vscode
+.vscode/
+
 telepresence.log

Makefile.d/proto.mk

@@ -58,8 +58,7 @@ proto/deps: \
 	$(GOPATH)/src/github.com/planetscale/vtprotobuf \
 	$(GOPATH)/src/github.com/protocolbuffers/protobuf \
 	$(GOPATH)/src/google.golang.org/genproto \
-	$(GOPATH)/src/google.golang.org/protobuf \
-	$(ROOTDIR)/apis/proto/v1
+	$(GOPATH)/src/google.golang.org/protobuf
 
 $(GOPATH)/src/github.com/protocolbuffers/protobuf:
 	git clone \

charts/vald/values.yaml

@@ -1715,6 +1715,9 @@ agent:
       # @schema {"name": "agent.ngt.kvsdb.concurrency", "type": "integer"}
       # agent.ngt.kvsdb.concurrency -- kvsdb processing concurrency
       concurrency: 6
+    # @schema {"name": "agent.ngt.broken_index_history_limit", "type": "integer"}
+    # agent.ngt.broken_index_history_limit -- maximum number of broken index generations to backup
+    broken_index_history_limit: 0
   # @schema {"name": "agent.sidecar", "type": "object"}
   sidecar:
     # @schema {"name": "agent.sidecar.enabled", "type": "boolean"}

cmd/agent/core/ngt/sample.yaml

@@ -126,3 +126,4 @@ ngt:
   object_type: float
   search_edge_size: 10
   enable_copy_on_write: false
+  broken_index_history_limit: 0

dockers/dev/Dockerfile

@@ -60,6 +60,11 @@ RUN make ngt/install \
     && make kubectl/install \
     && make k3d/install
 
+# change GOPATH owner to vscode to run commands as user vscode without root permission
+RUN chown -R vscode:vscode "${GOPATH}" \
+    # mkdir for agent configuration install
+    && mkdir -p /etc/server
+
 # k9s installs the binary to the current user which is root for devcontainer
 # so change the current user to vscode to user afterwards
 USER vscode

internal/config/ngt.go

@@ -88,6 +88,9 @@ type NGT struct {
 
 	// KVSDB represent the ngt bidirectional kv store configuration
 	KVSDB *KVSDB `json:"kvsdb,omitempty" yaml:"kvsdb"`
+
+	// BrokenIndexHistoryLimit represents the maximum number of broken index generations that will be backed up
+	BrokenIndexHistoryLimit int `yaml:"broken_index_history_limit" json:"broken_index_history_limit,omitempty"`
 }
 
 // KVSDB represent the ngt vector bidirectional kv store configuration

internal/core/algorithm/ngt/ngt.go

@@ -25,7 +25,6 @@ package ngt
 import "C"
 
 import (
-	"os"
 	"reflect"
 	"sync"
 	"unsafe"
@@ -307,18 +306,6 @@ func (n *ngt) loadOptions(opts ...Option) (err error) {
 }
 
 func (n *ngt) create() (err error) {
-	files, err := file.ListInDir(n.idxPath)
-	if err == nil && len(files) != 0 {
-		log.Warnf("index path exists, will remove the directories: %v", files)
-		for _, f := range files {
-			err = os.RemoveAll(f)
-			if err != nil {
-				return err
-			}
-		}
-	} else if err != nil {
-		log.Debug(err)
-	}
 	path := C.CString(n.idxPath)
 	defer C.free(unsafe.Pointer(path))
 

internal/errors/errors.go

@@ -80,6 +80,11 @@ var (
 		return Wrapf(err, "failed to output %s logs", str)
 	}
 
+	// ErrIndexPathNotExists represents a function to generate an error that the index path is not exists.
+	ErrIndexPathNotExists = func(path string) error {
+		return Errorf("index path %s not exists", path)
+	}
+
 	// New represents a function to generate the new error with a message.
 	// When the message is nil, it will return nil instead of an error.
 	New = func(msg string) error {

internal/test/data/agent/ngt/validIndex/README.md

@@ -0,0 +1 @@
+This directory holds an outcome of inserting `datafashion-mnist-784-euclidean` to Vald agent by using `example/client/agent/main.go` with `insertCount = 100`.

internal/test/data/agent/ngt/validIndex/metadata.json

@@ -0,0 +1 @@
+{ "is_invalid": false, "ngt": { "index_count": 100 } }

internal/test/data/agent/ngt/validIndex/prf

@@ -0,0 +1,26 @@
+AccuracyTable	
+BatchSizeForCreation	200
+BuildTimeLimit	0
+DatabaseType	Memory
+Dimension	784
+DistanceType	L2
+DynamicEdgeSizeBase	30
+DynamicEdgeSizeRate	20
+EdgeSizeForCreation	20
+EdgeSizeForSearch	10
+EdgeSizeLimitForCreation	5
+EpsilonForCreation	0.1
+GraphType	ANNG
+IncomingEdge	80
+IncrimentalEdgeSizeLimitForTruncation	0
+IndexType	GraphAndTree
+ObjectAlignment	False
+ObjectType	Float-4
+OutgoingEdge	10
+PathAdjustmentInterval	0
+PrefetchOffset	1
+PrefetchSize	3136
+SeedSize	10
+SeedType	None
+ThreadPoolSize	32
+TruncationThreadPoolSize	8

internal/test/testdata.go

@@ -21,6 +21,10 @@ import (
 	"github.com/vdaas/vald/internal/strings"
 )
 
+const (
+	ValidIndex = "agent/ngt/validIndex"
+)
+
 // GetTestdataPath returns the test data file path under `internal/test/data`.
 func GetTestdataPath(filename string) string {
 	return file.Join(baseDir(), "/internal/test/data/", filename)