Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor and add test for service/ngt.go #2040

Merged
merged 9 commits into from
May 26, 2023
Merged

refactor and add test for service/ngt.go #2040

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8386045
refactor New function
ykadowak May 24, 2023
bff00cc
add test for New
ykadowak May 24, 2023
49fcdd5
add success test for prepareFolders
ykadowak May 24, 2023
8aeb469
remove placeholder test
ykadowak May 24, 2023
16e1248
fix for linter
ykadowak May 24, 2023
6ad95a7
style: Format code with gofumpt and prettier
deepsource-autofix[bot] May 24, 2023
db561f2
fix
ykadowak May 25, 2023
5812658
remove file.Exists and add some warn msg
ykadowak May 25, 2023
0c1b1f8
Update pkg/agent/core/ngt/service/ngt.go
ykadowak May 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 32 additions & 21 deletions pkg/agent/core/ngt/service/ngt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,30 +159,13 @@ func New(cfg *config.NGT, opts ...Option) (nn NGT, err error) {
}
}
if len(n.path) == 0 {
log.Info("index path setting is empty, starting vald agent with in-memory mode")
n.inMem = true
}

if n.enableCopyOnWrite && !n.inMem && len(n.path) != 0 {
sep := string(os.PathSeparator)
n.path, err = filepath.Abs(strings.ReplaceAll(n.path, sep+sep, sep))
if err != nil {
log.Warn(err)
}
n.basePath = n.path
n.oldPath = file.Join(n.basePath, oldIndexDirName)
n.path = file.Join(n.basePath, originIndexDirName)
err = file.MkdirAll(n.oldPath, fs.ModePerm)
if err != nil {
log.Warn(err)
}
err = file.MkdirAll(n.path, fs.ModePerm)
if err != nil {
log.Warn(err)
}
err = n.mktmp()
if err != nil {
return nil, err
}
err = n.prepareFolders()
if err != nil {
return nil, err
}

err = n.initNGT(
Expand Down Expand Up @@ -215,6 +198,34 @@ func New(cfg *config.NGT, opts ...Option) (nn NGT, err error) {
return n, nil
}

func (n *ngt) prepareFolders() (err error) {
if n.enableCopyOnWrite && !n.inMem && len(n.path) != 0 {
sep := string(os.PathSeparator)
absPath, err := filepath.Abs(strings.ReplaceAll(n.path, sep+sep, sep))
if err != nil {
log.Warn("keep going with relative path: %w", err)
ykadowak marked this conversation as resolved.
Show resolved Hide resolved
} else {
n.path = absPath
}
n.basePath = n.path
n.oldPath = file.Join(n.basePath, oldIndexDirName)
n.path = file.Join(n.basePath, originIndexDirName)
err = file.MkdirAll(n.oldPath, fs.ModePerm)
if err != nil {
log.Warn(err)
}
err = file.MkdirAll(n.path, fs.ModePerm)
if err != nil {
log.Warn(err)
}
err = n.mktmp()
if err != nil {
return err
}
}
return nil
}

func (n *ngt) load(ctx context.Context, path string, opts ...core.Option) (err error) {
exist, fi, err := file.ExistsWithDetail(path)
switch {
Expand Down
265 changes: 173 additions & 92 deletions pkg/agent/core/ngt/service/ngt_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
core "github.com/vdaas/vald/internal/core/algorithm/ngt"
"github.com/vdaas/vald/internal/errgroup"
"github.com/vdaas/vald/internal/errors"
"github.com/vdaas/vald/internal/file"
"github.com/vdaas/vald/internal/log"
"github.com/vdaas/vald/internal/safety"
"github.com/vdaas/vald/internal/test/data/vector"
Expand All @@ -49,6 +50,178 @@ type index struct {
vec []float32
}

func TestNew(t *testing.T) {
type args struct {
cfg *config.NGT
opts []Option
}
type want struct {
err error
}
type test struct {
name string
args args
want want
checkFunc func(want, error) error
beforeFunc func(*testing.T, args)
afterFunc func(*testing.T, args)
}
defaultCheckFunc := func(w want, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
return nil
}
tests := []test{
func() test {
tmpDir := t.TempDir()
return test{
name: "success with default options",
args: args{
cfg: &config.NGT{
Dimension: 100,
DistanceType: "l2",
ObjectType: "float",
BulkInsertChunkSize: 10,
CreationEdgeSize: 20,
SearchEdgeSize: 10,
EnableProactiveGC: false,
EnableCopyOnWrite: false,
KVSDB: &config.KVSDB{
Concurrency: 10,
},
},
opts: []Option{
WithIndexPath(tmpDir),
},
},
want: want{
err: nil,
},
checkFunc: defaultCheckFunc,
}
}(),
}

for _, tc := range tests {
test := tc
t.Run(test.name, func(tt *testing.T) {
tt.Parallel()
defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
if test.beforeFunc != nil {
test.beforeFunc(tt, test.args)
}
if test.afterFunc != nil {
defer test.afterFunc(tt, test.args)
}
checkFunc := test.checkFunc
if test.checkFunc == nil {
checkFunc = defaultCheckFunc
}

_, err := New(test.args.cfg, test.args.opts...)
if err := checkFunc(test.want, err); err != nil {
tt.Errorf("error = %v", err)
}
})
}
}

func Test_ngt_prepareFolders(t *testing.T) {
type args struct{}
type want struct {
err error
}
type fields struct {
enableCopyOnWrite bool
path string
}
type test struct {
name string
args args
fields fields
want want
checkFunc func(want, error) error
beforeFunc func(*testing.T, args)
afterFunc func(*testing.T, args)
}
defaultCheckFunc := func(w want, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
return nil
}

tests := []test{
func() test {
tmpDir := t.TempDir()
return test{
name: "success to create origin and backup dir",
args: args{},
fields: fields{
enableCopyOnWrite: true,
path: tmpDir,
},
want: want{
err: nil,
},
checkFunc: func(w want, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
dirs, err := file.ListInDir(tmpDir)
if err != nil {
return err
}

// extract folder name from dir path into a map
dirMap := make(map[string]struct{}, len(dirs))
for _, dir := range dirs {
// extract folder name from dir path
dir = dir[len(tmpDir)+1:]
dirMap[dir] = struct{}{}
}

// check if the dirs slice contains origin or backup.
// if the dirs slice contains both origin and backup, it is an error.
if _, ok := dirMap[originIndexDirName]; !ok {
return fmt.Errorf("failed to create origin dir")
}
if _, ok := dirMap[oldIndexDirName]; !ok {
return fmt.Errorf("failed to create backup dir")
}
return nil
},
}
}(),
}
for _, tc := range tests {
test := tc
t.Run(test.name, func(tt *testing.T) {
tt.Parallel()
defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
if test.beforeFunc != nil {
test.beforeFunc(tt, test.args)
}
if test.afterFunc != nil {
defer test.afterFunc(tt, test.args)
}
checkFunc := test.checkFunc
if test.checkFunc == nil {
checkFunc = defaultCheckFunc
}
n := &ngt{
enableCopyOnWrite: test.fields.enableCopyOnWrite,
path: test.fields.path,
}
err := n.prepareFolders()
if err := checkFunc(test.want, err); err != nil {
tt.Errorf("error = %v", err)
}
})
}
}

func Test_ngt_InsertUpsert(t *testing.T) {
if testing.Short() {
t.Skip("The execution of this test takes a lot of time, so it is not performed during the short test\ttest: Test_ngt_InsertUpsert")
Expand Down Expand Up @@ -395,98 +568,6 @@ func createRandomData(num int, cfg *createRandomDataConfig) []index {

// NOT IMPLEMENTED BELOW

func TestNew(t *testing.T) {
type args struct {
cfg *config.NGT
opts []Option
}
type want struct {
wantNn NGT
err error
}
type test struct {
name string
args args
want want
checkFunc func(want, NGT, error) error
beforeFunc func(*testing.T, args)
afterFunc func(*testing.T, args)
}
defaultCheckFunc := func(w want, gotNn NGT, err error) error {
if !errors.Is(err, w.err) {
return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err)
}
if !reflect.DeepEqual(gotNn, w.wantNn) {
return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotNn, w.wantNn)
}
return nil
}
tests := []test{
// TODO test cases
/*
{
name: "test_case_1",
args: args {
cfg:nil,
opts:nil,
},
want: want{},
checkFunc: defaultCheckFunc,
beforeFunc: func(t *testing.T, args args) {
t.Helper()
},
afterFunc: func(t *testing.T, args args) {
t.Helper()
},
},
*/

// TODO test cases
/*
func() test {
return test {
name: "test_case_2",
args: args {
cfg:nil,
opts:nil,
},
want: want{},
checkFunc: defaultCheckFunc,
beforeFunc: func(t *testing.T, args args) {
t.Helper()
},
afterFunc: func(t *testing.T, args args) {
t.Helper()
},
}
}(),
*/
}

for _, tc := range tests {
test := tc
t.Run(test.name, func(tt *testing.T) {
tt.Parallel()
defer goleak.VerifyNone(tt, goleak.IgnoreCurrent())
if test.beforeFunc != nil {
test.beforeFunc(tt, test.args)
}
if test.afterFunc != nil {
defer test.afterFunc(tt, test.args)
}
checkFunc := test.checkFunc
if test.checkFunc == nil {
checkFunc = defaultCheckFunc
}

gotNn, err := New(test.args.cfg, test.args.opts...)
if err := checkFunc(test.want, gotNn, err); err != nil {
tt.Errorf("error = %v", err)
}
})
}
}

func Test_ngt_load(t *testing.T) {
type args struct {
ctx context.Context
Expand Down