[patch] divide gateway for future architecture

go.mod

@@ -64,7 +64,7 @@ require (
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed
-	golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea // indirect
+	golang.org/x/tools v0.0.0-20200811153730-74512f09e4b0 // indirect
 	gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614
 	gonum.org/v1/plot v0.7.1-0.20200803120916-6a037fda5e90
 	google.golang.org/api v0.30.1-0.20200811142409-9e2b1a1ddef6

go.sum

@@ -871,8 +871,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200809012840-6f4f008689da h1:ml5G98G4/tdKT1XNq+ky5iSRdKKux0TANlLAzmXT/hg=
 golang.org/x/tools v0.0.0-20200809012840-6f4f008689da/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea h1:9ym67RBRK/wN50W0T3g8g1n8viM1D2ofgWufDlMfWe0=
-golang.org/x/tools v0.0.0-20200811032001-fd80f4dbb3ea/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200811153730-74512f09e4b0 h1:UQpfk6CmZq0nGdlZ3/lNwnKmVvi0mgAZCyW6L2aZj/Y=
+golang.org/x/tools v0.0.0-20200811153730-74512f09e4b0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/config/config.go

@@ -19,13 +19,13 @@ package config
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 	"unsafe"
 
 	"github.com/vdaas/vald/internal/encoding/json"
+	"github.com/vdaas/vald/internal/io/ioutil"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -100,13 +100,7 @@ func GetActualValue(val string) (res string) {
 	}
 	res = os.ExpandEnv(val)
 	if strings.HasPrefix(res, fileValuePrefix) {
-		path := strings.TrimPrefix(res, fileValuePrefix)
-		file, err := os.OpenFile(path, os.O_RDONLY, 0600)
-		defer file.Close()
-		if err != nil {
-			return
-		}
-		body, err := ioutil.ReadAll(file)
+		body, err := ioutil.ReadFile(strings.TrimPrefix(res, fileValuePrefix))
 		if err != nil {
 			return
 		}

internal/db/storage/blob/s3/reader/reader.go

@@ -117,8 +117,9 @@ func (r *reader) Open(ctx context.Context) (err error) {
 }
 
 func (r *reader) getObjectWithBackoff(ctx context.Context, offset, length int64) (io.Reader, error) {
-	getFunc := func() (interface{}, error) {
-		return r.getObject(ctx, offset, length)
+	getFunc := func() (interface{}, bool, error) {
+		res, err := r.getObject(ctx, offset, length)
+		return res, err != nil, err
 	}
 
 	b := backoff.New(r.backoffOpts...)

internal/io/io_test.go

@@ -29,6 +29,7 @@ import (
 )
 
 func TestNewReaderWithContext(t *testing.T) {
+	t.Parallel()
 	type args struct {
 		ctx context.Context
 		r   io.Reader
@@ -86,6 +87,7 @@ func TestNewReaderWithContext(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc(test.args)
@@ -107,6 +109,7 @@ func TestNewReaderWithContext(t *testing.T) {
 }
 
 func TestNewReadCloserWithContext(t *testing.T) {
+	t.Parallel()
 	type args struct {
 		ctx context.Context
 		r   io.ReadCloser
@@ -164,6 +167,7 @@ func TestNewReadCloserWithContext(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc(test.args)
@@ -185,6 +189,7 @@ func TestNewReadCloserWithContext(t *testing.T) {
 }
 
 func Test_ctxReader_Read(t *testing.T) {
+	t.Parallel()
 	type args struct {
 		p []byte
 	}
@@ -252,6 +257,7 @@ func Test_ctxReader_Read(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc(test.args)
@@ -277,6 +283,7 @@ func Test_ctxReader_Read(t *testing.T) {
 }
 
 func Test_ctxReader_Close(t *testing.T) {
+	t.Parallel()
 	type fields struct {
 		ctx context.Context
 		r   io.Reader
@@ -330,6 +337,7 @@ func Test_ctxReader_Close(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc()
@@ -355,6 +363,7 @@ func Test_ctxReader_Close(t *testing.T) {
 }
 
 func TestNewWriterWithContext(t *testing.T) {
+	t.Parallel()
 	type args struct {
 		ctx context.Context
 	}
@@ -413,6 +422,7 @@ func TestNewWriterWithContext(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc(test.args)
@@ -435,6 +445,7 @@ func TestNewWriterWithContext(t *testing.T) {
 }
 
 func TestNewWriteCloserWithContext(t *testing.T) {
+	t.Parallel()
 	type args struct {
 		ctx context.Context
 		w   io.WriteCloser
@@ -492,6 +503,7 @@ func TestNewWriteCloserWithContext(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc(test.args)
@@ -513,6 +525,7 @@ func TestNewWriteCloserWithContext(t *testing.T) {
 }
 
 func Test_ctxWriter_Write(t *testing.T) {
+	t.Parallel()
 	type args struct {
 		p []byte
 	}
@@ -580,6 +593,7 @@ func Test_ctxWriter_Write(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc(test.args)
@@ -605,6 +619,7 @@ func Test_ctxWriter_Write(t *testing.T) {
 }
 
 func Test_ctxWriter_Close(t *testing.T) {
+	t.Parallel()
 	type fields struct {
 		ctx context.Context
 		w   io.Writer
@@ -658,6 +673,7 @@ func Test_ctxWriter_Close(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
 			defer goleak.VerifyNone(tt)
 			if test.beforeFunc != nil {
 				test.beforeFunc()

internal/io/ioutil/ioutil.go

@@ -0,0 +1,52 @@
+//
+// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt )
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package ioutil provides utility function for I/O
+package ioutil
+
+import (
+	"bytes"
+	"os"
+
+	"github.com/vdaas/vald/internal/safety"
+)
+
+func ReadFile(path string) ([]byte, error) {
+	f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var n int64 = bytes.MinRead
+	if fi, err := f.Stat(); err == nil {
+		if size := fi.Size() + bytes.MinRead; size > n {
+			n = size
+		}
+	}
+
+	buf := bytes.NewBuffer(make([]byte, 0, n))
+
+	err = safety.RecoverFunc(func() (err error) {
+		_, err = buf.ReadFrom(f)
+		return err
+	})()
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

internal/io/ioutil/ioutil_test.go

@@ -0,0 +1,103 @@
+//
+// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt )
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package ioutil provides utility function for I/O
+package ioutil
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/vdaas/vald/internal/errors"
+	"go.uber.org/goleak"
+)
+
+func TestReadFile(t *testing.T) {
+	t.Parallel()
+	type args struct {
+		path string
+	}
+	type want struct {
+		want []byte
+		err  error
+	}
+	type test struct {
+		name       string
+		args       args
+		want       want
+		checkFunc  func(want, []byte, error) error
+		beforeFunc func(args)
+		afterFunc  func(args)
+	}
+	defaultCheckFunc := func(w want, got []byte, err error) error {
+		if !errors.Is(err, w.err) {
+			return errors.Errorf("got error = %v, want %v", err, w.err)
+		}
+		if !reflect.DeepEqual(got, w.want) {
+			return errors.Errorf("got = %v, want %v", got, w.want)
+		}
+		return nil
+	}
+	tests := []test{
+		// TODO test cases
+		/*
+		   {
+		       name: "test_case_1",
+		       args: args {
+		           path: "",
+		       },
+		       want: want{},
+		       checkFunc: defaultCheckFunc,
+		   },
+		*/
+
+		// TODO test cases
+		/*
+		   func() test {
+		       return test {
+		           name: "test_case_2",
+		           args: args {
+		           path: "",
+		           },
+		           want: want{},
+		           checkFunc: defaultCheckFunc,
+		       }
+		   }(),
+		*/
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(tt *testing.T) {
+			tt.Parallel()
+			defer goleak.VerifyNone(tt)
+			if test.beforeFunc != nil {
+				test.beforeFunc(test.args)
+			}
+			if test.afterFunc != nil {
+				defer test.afterFunc(test.args)
+			}
+			if test.checkFunc == nil {
+				test.checkFunc = defaultCheckFunc
+			}
+
+			got, err := ReadFile(test.args.path)
+			if err := test.checkFunc(test.want, got, err); err != nil {
+				tt.Errorf("error = %v", err)
+			}
+
+		})
+	}
+}

internal/tls/tls.go

@@ -18,14 +18,12 @@
 package tls
 
 import (
-	"bytes"
 	"crypto/tls"
 	"crypto/x509"
-	"os"
 	"reflect"
 
 	"github.com/vdaas/vald/internal/errors"
-	"github.com/vdaas/vald/internal/safety"
+	"github.com/vdaas/vald/internal/io/ioutil"
 )
 
 type Config = tls.Config
@@ -105,28 +103,10 @@ func NewClientConfig(opts ...Option) (*Config, error) {
 // NewX509CertPool returns *x509.CertPool struct or error.
 // The CertPool will read the certificate from the path, and append the content to the system certificate pool, and return.
 func NewX509CertPool(path string) (pool *x509.CertPool, err error) {
-	f, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
+	c, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
-	defer f.Close()
-
-	var n int64 = bytes.MinRead
-	if fi, err := f.Stat(); err == nil {
-		if size := fi.Size() + bytes.MinRead; size > n {
-			n = size
-		}
-	}
-
-	buf := bytes.NewBuffer(make([]byte, 0, n))
-	err = safety.RecoverFunc(func() (err error) {
-		_, err = buf.ReadFrom(f)
-		return err
-	})()
-	if err != nil {
-		return nil, err
-	}
-	c := buf.Bytes()
 	if err == nil && c != nil {
 		pool, err = x509.SystemCertPool()
 		if err != nil || pool == nil {