Add backup origin when CoW enabled and failed to load primary (#2091)

* add backup origin when CoW enabled and failed to load primary

* remove TODO comment

* refactor errors

* Refactor log message in ngt.go to use %s instead of %v

* fix erros.Join args order

internal/errors/agent.go

@@ -109,4 +109,7 @@ var (
 	ErrAgentMigrationFailed = func(err error) error {
 		return Wrap(err, "index_path migration failed")
 	}
+
+	// ErrAgentIndexDirectoryRecreationFailed represents an error that the index directory recreation failed during the process of broken index backup.
+	ErrIndexDirectoryRecreationFailed = New("failed to recreate the index directory")
 )

pkg/agent/core/ngt/service/ngt.go

@@ -527,6 +527,11 @@ func (n *ngt) backupBroken(ctx context.Context) error {
 	atomic.SwapUint64(&n.nobic, uint64(len(files)))
 	log.Debugf("broken index count updated: %v", n.nobic)
 
+	// remake the path since it has been moved to broken directory
+	if err := file.MkdirAll(n.path, fs.ModePerm); err != nil {
+		return errors.Join(err, errors.ErrIndexDirectoryRecreationFailed)
+	}
+
 	return nil
 }
 
@@ -586,12 +591,6 @@ func (n *ngt) rebuild(ctx context.Context, path string, opts ...core.Option) (er
 		err = n.backupBroken(ctx)
 		if err != nil {
 			log.Warnf("failed to backup broken index. will remove it and restart: %v", err)
-		} else {
-			// remake the path since it has been moved to broken directory
-			err = file.MkdirAll(n.path, fs.ModePerm)
-			if err != nil {
-				return fmt.Errorf("failed to recreate the index directory: %w", err)
-			}
 		}
 	}
 
@@ -655,6 +654,12 @@ func (n *ngt) initNGT(opts ...core.Option) (err error) {
 			)
 		} else {
 			log.Warnf("failed to load vald primary index from %s\t error: %v\ttrying to load from old copied index data from %s", n.path, err, n.oldPath)
+			if needsBackup(n.path) {
+				log.Infof("starting to backup broken index at %s", n.path)
+				if err := n.backupBroken(ctx); err != nil {
+					log.Warnf("failed to backup broken index. will try to restart from old index anyway: %v", err)
+				}
+			}
 		}
 	} else {
 		return nil

pkg/agent/core/ngt/service/ngt_test.go

@@ -518,6 +518,78 @@ func TestNew(t *testing.T) {
 						return fmt.Errorf("broken index directory is not empty")
 					}
 
+					return nil
+				},
+			}
+		}(),
+		func() test {
+			tmpDir := t.TempDir()
+			originDir := filepath.Join(tmpDir, originIndexDirName)
+			backupDir := filepath.Join(tmpDir, oldIndexDirName)
+			brokenDir := filepath.Join(tmpDir, brokenIndexDirName)
+			testIndexDir := testdata.GetTestdataPath(testdata.ValidIndex)
+			config := defaultConfig
+			config.BrokenIndexHistoryLimit = 1
+			config.EnableCopyOnWrite = true
+			return test{
+				name: "New backup broken index when CoW is enabled and failed to load primary index",
+				args: args{
+					cfg: &config,
+					opts: []Option{
+						WithIndexPath(tmpDir),
+					},
+				},
+				want: want{
+					err: nil,
+				},
+				beforeFunc: func(t *testing.T, args args) {
+					t.Helper()
+					if err := file.MkdirAll(originDir, fs.ModePerm); err != nil {
+						t.Errorf("failed to create origin dir: %v", err)
+					}
+					if err := file.CopyDir(context.Background(), testIndexDir, originDir); err != nil {
+						t.Errorf("failed to copy test index: %v", err)
+					}
+					// remove metadata.json to make it broken
+					if err := os.Remove(filepath.Join(originDir, "metadata.json")); err != nil {
+						t.Errorf("failed to remove index file: %v", err)
+					}
+
+					if err := file.MkdirAll(backupDir, fs.ModePerm); err != nil {
+						t.Errorf("failed to create backup dir: %v", err)
+					}
+					if err := file.CopyDir(context.Background(), testIndexDir, backupDir); err != nil {
+						t.Errorf("failed to copy test index: %v", err)
+					}
+				},
+				checkFunc: func(w want, err error) error {
+					if !errors.Is(err, w.err) {
+						return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
+					}
+					files, err := file.ListInDir(brokenDir)
+					if err != nil {
+						return err
+					}
+					if len(files) != 1 {
+						return fmt.Errorf("only one generation should be in broken dir but there's %v", len(files))
+					}
+
+					broken, err := file.ListInDir(files[0])
+					if err != nil {
+						return err
+					}
+					if len(broken) == 0 {
+						return fmt.Errorf("failed to move broken index files")
+					}
+
+					files, err = file.ListInDir(originDir)
+					if err != nil {
+						return err
+					}
+					if len(files) != 0 {
+						return fmt.Errorf("failed to move origin index files to broken directory")
+					}
+
 					return nil
 				},
 			}