diff --git a/internal/db/storage/blob/s3/option.go b/internal/db/storage/blob/s3/option.go index 712c4d8a1c..86de4bb86a 100644 --- a/internal/db/storage/blob/s3/option.go +++ b/internal/db/storage/blob/s3/option.go @@ -24,6 +24,7 @@ import ( "github.com/vdaas/vald/internal/unit" ) +// Option represents the functional option for client. type Option func(c *client) error var ( @@ -32,6 +33,7 @@ var ( } ) +// WithErrGroup returns the option to set the eg. func WithErrGroup(eg errgroup.Group) Option { return func(c *client) error { if eg != nil { @@ -41,6 +43,7 @@ func WithErrGroup(eg errgroup.Group) Option { } } +// WithSession returns the option to set the session. func WithSession(sess *session.Session) Option { return func(c *client) error { if sess != nil { @@ -50,6 +53,7 @@ func WithSession(sess *session.Session) Option { } } +// WithBucket returns the option to set bucket. func WithBucket(bucket string) Option { return func(c *client) error { c.bucket = bucket @@ -57,6 +61,7 @@ func WithBucket(bucket string) Option { } } +// WithMaxPartSize returns the option to set maxPartSize. func WithMaxPartSize(size string) Option { return func(c *client) error { b, err := unit.ParseBytes(size) @@ -72,6 +77,7 @@ func WithMaxPartSize(size string) Option { } } +// WithMaxChunkSize returns the option to set maxChunkSize. func WithMaxChunkSize(size string) Option { return func(c *client) error { b, err := unit.ParseBytes(size) @@ -87,6 +93,7 @@ func WithMaxChunkSize(size string) Option { } } +// WithReaderBackoff returns the option to set readerBackoffEnabled. func WithReaderBackoff(enabled bool) Option { return func(c *client) error { c.readerBackoffEnabled = enabled @@ -94,13 +101,18 @@ func WithReaderBackoff(enabled bool) Option { } } +// WithReaderBackoffOpts returns the option to set readerBackoffOpts. func WithReaderBackoffOpts(opts ...backoff.Option) Option { return func(c *client) error { - if c.readerBackoffOpts == nil { - c.readerBackoffOpts = opts + if opts == nil { + return nil + } + if c.readerBackoffOpts != nil { + c.readerBackoffOpts = append(c.readerBackoffOpts, opts...) + return nil } - c.readerBackoffOpts = append(c.readerBackoffOpts, opts...) + c.readerBackoffOpts = opts return nil } diff --git a/internal/db/storage/blob/s3/option_test.go b/internal/db/storage/blob/s3/option_test.go index da7b728e3c..81faf04d81 100644 --- a/internal/db/storage/blob/s3/option_test.go +++ b/internal/db/storage/blob/s3/option_test.go @@ -17,91 +17,87 @@ package s3 import ( + stderrs "errors" + "reflect" "testing" "github.com/aws/aws-sdk-go/aws/session" + "github.com/google/go-cmp/cmp" + "github.com/vdaas/vald/internal/backoff" "github.com/vdaas/vald/internal/errgroup" + "github.com/vdaas/vald/internal/errors" "go.uber.org/goleak" ) +var ( + // Goroutine leak is detected by `fastime`, but it should be ignored in the test because it is an external package. + goleakIgnoreOptions = []goleak.Option{ + goleak.IgnoreTopFunction("github.com/kpango/fastime.(*Fastime).StartTimerD.func1"), + } +) + func TestWithErrGroup(t *testing.T) { - // Change interface type to the type of object you are testing - type T = interface{} + type T = client type args struct { eg errgroup.Group } type want struct { obj *T - // Uncomment this line if the option returns an error, otherwise delete it - // err error + err error } type test struct { - name string - args args - want want - // Use the first line if the option returns an error. otherwise use the second line - // checkFunc func(want, *T, error) error - // checkFunc func(want, *T) error + name string + args args + want want + checkFunc func(want, *T, error) error beforeFunc func(args) afterFunc func(args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T, err error) error { - if !errors.Is(err, w.err) { - return errors.Errorf("got error = %v, want %v", err, w.err) - } - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ - - // Uncomment this block if the option do not returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T) error { - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - eg: nil, - }, - want: want { - obj: new(T), - }, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - eg: nil, - }, - want: want { - obj: new(T), - }, - } - }(), - */ + func() test { + eg := errgroup.Get() + return test{ + name: "set success when eg is not nil", + args: args{ + eg: eg, + }, + want: want{ + obj: &T{ + eg: eg, + }, + }, + } + }(), + + func() test { + return test{ + name: "set nothing when eg is nil", + args: args{ + eg: nil, + }, + want: want{ + obj: &T{ + eg: nil, + }, + }, + } + }(), } for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(tt) + defer goleak.VerifyNone(tt, goleakIgnoreOptions...) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -109,113 +105,81 @@ func TestWithErrGroup(t *testing.T) { defer test.afterFunc(test.args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - - got := WithErrGroup(test.args.eg) - obj := new(T) - if err := test.checkFunc(test.want, obj, got(obj)); err != nil { - tt.Errorf("error = %v", err) - } - */ - - // Uncomment this block if the option do not return an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - got := WithErrGroup(test.args.eg) - obj := new(T) - got(obj) - if err := test.checkFunc(test.want, obj); err != nil { - tt.Errorf("error = %v", err) - } - */ + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithErrGroup(test.args.eg) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } }) } } func TestWithSession(t *testing.T) { - // Change interface type to the type of object you are testing - type T = interface{} + type T = client type args struct { sess *session.Session } type want struct { obj *T - // Uncomment this line if the option returns an error, otherwise delete it - // err error + err error } type test struct { - name string - args args - want want - // Use the first line if the option returns an error. otherwise use the second line - // checkFunc func(want, *T, error) error - // checkFunc func(want, *T) error + name string + args args + want want + checkFunc func(want, *T, error) error beforeFunc func(args) afterFunc func(args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T, err error) error { - if !errors.Is(err, w.err) { - return errors.Errorf("got error = %v, want %v", err, w.err) - } - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ - - // Uncomment this block if the option do not returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T) error { - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - sess: nil, - }, - want: want { - obj: new(T), - }, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - sess: nil, - }, - want: want { - obj: new(T), - }, - } - }(), - */ + func() test { + sess := new(session.Session) + return test{ + name: "set success when sess is not nil", + args: args{ + sess: sess, + }, + want: want{ + obj: &T{ + session: sess, + }, + }, + } + }(), + + func() test { + return test{ + name: "set nothing when sess is not nil", + args: args{ + sess: nil, + }, + want: want{ + obj: &T{ + session: nil, + }, + }, + } + }(), } for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(tt) + defer goleak.VerifyNone(tt, goleakIgnoreOptions...) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -223,113 +187,64 @@ func TestWithSession(t *testing.T) { defer test.afterFunc(test.args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - - got := WithSession(test.args.sess) - obj := new(T) - if err := test.checkFunc(test.want, obj, got(obj)); err != nil { - tt.Errorf("error = %v", err) - } - */ - - // Uncomment this block if the option do not return an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - got := WithSession(test.args.sess) - obj := new(T) - got(obj) - if err := test.checkFunc(test.want, obj); err != nil { - tt.Errorf("error = %v", err) - } - */ + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithSession(test.args.sess) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } }) } } func TestWithBucket(t *testing.T) { - // Change interface type to the type of object you are testing - type T = interface{} + type T = client type args struct { bucket string } type want struct { obj *T - // Uncomment this line if the option returns an error, otherwise delete it - // err error + err error } type test struct { - name string - args args - want want - // Use the first line if the option returns an error. otherwise use the second line - // checkFunc func(want, *T, error) error - // checkFunc func(want, *T) error + name string + args args + want want + checkFunc func(want, *T, error) error beforeFunc func(args) afterFunc func(args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T, err error) error { - if !errors.Is(err, w.err) { - return errors.Errorf("got error = %v, want %v", err, w.err) - } - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ - - // Uncomment this block if the option do not returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T) error { - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - bucket: "", - }, - want: want { - obj: new(T), - }, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - bucket: "", - }, - want: want { - obj: new(T), - }, - } - }(), - */ + { + name: "set success when bucket is `bucket`", + args: args{ + bucket: "bucket", + }, + want: want{ + obj: &T{ + bucket: "bucket", + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(tt) + defer goleak.VerifyNone(tt, goleakIgnoreOptions...) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -337,113 +252,189 @@ func TestWithBucket(t *testing.T) { defer test.afterFunc(test.args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - - got := WithBucket(test.args.bucket) - obj := new(T) - if err := test.checkFunc(test.want, obj, got(obj)); err != nil { - tt.Errorf("error = %v", err) - } - */ - - // Uncomment this block if the option do not return an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - got := WithBucket(test.args.bucket) - obj := new(T) - got(obj) - if err := test.checkFunc(test.want, obj); err != nil { - tt.Errorf("error = %v", err) - } - */ + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithBucket(test.args.bucket) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } }) } } func TestWithMaxPartSize(t *testing.T) { - // Change interface type to the type of object you are testing - type T = interface{} + type T = client + type args struct { + size string + } + type want struct { + obj *T + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, *T, error) error + beforeFunc func(args) + afterFunc func(args) + } + + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + + tests := []test{ + { + name: "set nothing when size is `100G`", + args: args{ + size: "100G", + }, + want: want{ + obj: &T{ + maxPartSize: 107374182400, + }, + }, + }, + + { + name: "set nothing when size is `1M`", + args: args{ + size: "1M", + }, + want: want{ + obj: &T{ + maxPartSize: 0, + }, + }, + }, + + { + name: "set nothing when size is `a`", + args: args{ + size: "a", + }, + want: want{ + obj: &T{ + maxPartSize: 0, + }, + err: func() (err error) { + err = stderrs.New("byte quantity must be a positive integer with a unit of measurement like M, MB, MiB, G, GiB, or GB") + err = errors.Wrap(err, errors.ErrParseUnitFailed("a").Error()) + return + }(), + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt, goleakIgnoreOptions...) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithMaxPartSize(test.args.size) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func TestWithMaxChunkSize(t *testing.T) { + type T = client type args struct { size string } type want struct { obj *T - // Uncomment this line if the option returns an error, otherwise delete it - // err error + err error } type test struct { - name string - args args - want want - // Use the first line if the option returns an error. otherwise use the second line - // checkFunc func(want, *T, error) error - // checkFunc func(want, *T) error + name string + args args + want want + checkFunc func(want, *T, error) error beforeFunc func(args) afterFunc func(args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T, err error) error { - if !errors.Is(err, w.err) { - return errors.Errorf("got error = %v, want %v", err, w.err) - } - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ - - // Uncomment this block if the option do not returns an error, otherwise delete it - /* - defaultCheckFunc := func(w want, obj *T) error { - if !reflect.DeepEqual(obj, w.obj) { - return errors.Errorf("got = %v, want %v", obj, w.obj) - } - return nil - } - */ + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } tests := []test{ - // TODO test cases - /* - { - name: "test_case_1", - args: args { - size: "", - }, - want: want { - obj: new(T), - }, - }, - */ - - // TODO test cases - /* - func() test { - return test { - name: "test_case_2", - args: args { - size: "", - }, - want: want { - obj: new(T), - }, - } - }(), - */ + { + name: "set nothing when size is `100G`", + args: args{ + size: "100G", + }, + want: want{ + obj: &T{ + maxChunkSize: 107374182400, + }, + err: nil, + }, + }, + + { + name: "set nothing when size is `1M`", + args: args{ + size: "1M", + }, + want: want{ + obj: &T{ + maxChunkSize: 0, + }, + err: nil, + }, + }, + + { + name: "set nothing when size is `a`", + args: args{ + size: "a", + }, + want: want{ + obj: &T{ + maxChunkSize: 0, + }, + err: func() (err error) { + err = stderrs.New("byte quantity must be a positive integer with a unit of measurement like M, MB, MiB, G, GiB, or GB") + err = errors.Wrap(err, errors.ErrParseUnitFailed("a").Error()) + return + }(), + }, + }, } for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - defer goleak.VerifyNone(tt) + defer goleak.VerifyNone(tt, goleakIgnoreOptions...) if test.beforeFunc != nil { test.beforeFunc(test.args) } @@ -451,31 +442,202 @@ func TestWithMaxPartSize(t *testing.T) { defer test.afterFunc(test.args) } - // Uncomment this block if the option returns an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - - got := WithMaxPartSize(test.args.size) - obj := new(T) - if err := test.checkFunc(test.want, obj, got(obj)); err != nil { - tt.Errorf("error = %v", err) - } - */ - - // Uncomment this block if the option do not return an error, otherwise delete it - /* - if test.checkFunc == nil { - test.checkFunc = defaultCheckFunc - } - got := WithMaxPartSize(test.args.size) - obj := new(T) - got(obj) - if err := test.checkFunc(test.want, obj); err != nil { - tt.Errorf("error = %v", err) - } - */ + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithMaxChunkSize(test.args.size) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func TestWithReaderBackoff(t *testing.T) { + type T = client + type args struct { + enabled bool + } + type want struct { + obj *T + err error + } + type test struct { + name string + args args + want want + checkFunc func(want, *T, error) error + beforeFunc func(args) + afterFunc func(args) + } + + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + if !reflect.DeepEqual(obj, w.obj) { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + return nil + } + + tests := []test{ + { + name: "set success when enabled is `true`", + args: args{ + enabled: true, + }, + want: want{ + obj: &T{ + readerBackoffEnabled: true, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt, goleakIgnoreOptions...) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithReaderBackoff(test.args.enabled) + obj := new(T) + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } + }) + } +} + +func TestWithReaderBackoffOpts(t *testing.T) { + type T = client + type args struct { + opts []backoff.Option + } + type fields struct { + readerBackoffOpts []backoff.Option + } + type want struct { + obj *T + err error + } + type test struct { + name string + args args + fields fields + want want + checkFunc func(want, *T, error) error + beforeFunc func(args) + afterFunc func(args) + } + + defaultCheckFunc := func(w want, obj *T, err error) error { + if !errors.Is(err, w.err) { + return errors.Errorf("got error = %v, want %v", err, w.err) + } + + opts := []cmp.Option{ + cmp.AllowUnexported(*obj), + cmp.AllowUnexported(*w.obj), + cmp.Comparer(func(want, got []backoff.Option) bool { + return len(got) == len(want) + }), + cmp.Comparer(func(want, got backoff.Option) bool { + return reflect.ValueOf(got).Pointer() == reflect.ValueOf(want).Pointer() + }), + } + if diff := cmp.Diff(w.obj, obj, opts...); diff != "" { + return errors.Errorf("got = %v, want %v", obj, w.obj) + } + + return nil + } + + tests := []test{ + { + name: "set success when opts is not nil and c.readerBackoffOpts is not nil", + args: args{ + opts: []backoff.Option{ + backoff.WithRetryCount(1), + }, + }, + fields: fields{ + readerBackoffOpts: []backoff.Option{ + backoff.WithRetryCount(1), + }, + }, + want: want{ + obj: &T{ + readerBackoffOpts: []backoff.Option{ + backoff.WithRetryCount(1), + backoff.WithRetryCount(1), + }, + }, + err: nil, + }, + }, + + { + name: "set success when opts is not nil and r.readerBackoffOpts is nil", + args: args{ + opts: []backoff.Option{ + backoff.WithRetryCount(1), + }, + }, + want: want{ + obj: &T{ + readerBackoffOpts: []backoff.Option{ + backoff.WithRetryCount(1), + }, + }, + err: nil, + }, + }, + + { + name: "set nothing when opts is nil", + args: args{ + opts: nil, + }, + want: want{ + obj: new(T), + err: nil, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + defer goleak.VerifyNone(tt, goleakIgnoreOptions...) + if test.beforeFunc != nil { + test.beforeFunc(test.args) + } + if test.afterFunc != nil { + defer test.afterFunc(test.args) + } + + if test.checkFunc == nil { + test.checkFunc = defaultCheckFunc + } + + got := WithReaderBackoffOpts(test.args.opts...) + obj := &T{ + readerBackoffOpts: test.fields.readerBackoffOpts, + } + if err := test.checkFunc(test.want, obj, got(obj)); err != nil { + tt.Errorf("error = %v", err) + } }) } }