services: Implement GSP-90: Re-support Initialization Via Connection …

…String (#589)

* parse pairs

* remove debug log

* fix service pair internal name, add tests

* update
- PairMap: reflect.Type -> string
- move from package pairs to services
- only expose NewServicerFromMap, hide parseMap

* - generate pairMap variable, remove PairMap type
- merge global pairs into service pairMaps
- remove New*FromMap, add New*FromString

* RegisterServicePairMap -> RegisterServiceSchema

* implement config string

* - rename config string -> connection string
- mark `RegisterServiceSchema` should not call by users
- remove parseListMode

* update test

* rename config to connStr

* ServiceSchema -> Schema

* - allow both <name> and <work_dir> missing
- test pairs order

* style updates
- variable work_dir -> workDir
- err1 -> parseErr
- merge parse_pairs.go into new.go
- parseString ->parseConnectionString

* defensive programming

cmd/definitions/tests/connstr_test.go

@@ -0,0 +1,137 @@
+package tests
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/beyondstorage/go-storage/v4/pairs"
+	"github.com/beyondstorage/go-storage/v4/services"
+	. "github.com/beyondstorage/go-storage/v4/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFromString(t *testing.T) {
+	cases := []struct {
+		name    string
+		connStr string
+		pairs   []Pair
+		err     error
+	}{
+		{
+			"empty",
+			"",
+			nil,
+			services.ErrConnectionStringInvalid,
+		},
+		{
+			"simplest",
+			"tests://",
+			nil,
+			nil,
+		},
+		{
+			"only options",
+			"tests://?size=200",
+			[]Pair{
+				pairs.WithSize(200),
+			},
+			nil,
+		},
+		{
+			"only root dir",
+			"tests:///",
+			[]Pair{
+				pairs.WithWorkDir("/"),
+			},
+			nil,
+		},
+		{
+			"end with ?",
+			"tests:///?",
+			[]Pair{
+				pairs.WithWorkDir("/"),
+			},
+			nil,
+		},
+		{
+			"stupid, but valid (ignored)",
+			"tests:///?&??&&&",
+			[]Pair{
+				pairs.WithWorkDir("/"),
+			},
+			nil,
+		},
+		{
+			"value can contain all characters except &",
+			"tests:///?string_pair=a=b:/c?d&size=200",
+			[]Pair{
+				pairs.WithWorkDir("/"),
+				WithStringPair("a=b:/c?d"),
+				pairs.WithSize(200),
+			},
+			nil,
+		},
+		{
+			"full format",
+			"tests://abc/tmp/tmp1?size=200&expire=100&storage_class=sc",
+			[]Pair{
+				pairs.WithName("abc"),
+				pairs.WithWorkDir("/tmp/tmp1"),
+				pairs.WithSize(200),
+				pairs.WithExpire(100),
+				WithStorageClass("sc"),
+			},
+			nil,
+		},
+		{
+			"duplicate key, appear in order (finally, first will be picked)",
+			"tests://abc/tmp/tmp1?size=200&name=def&size=300",
+			[]Pair{
+				pairs.WithName("abc"),
+				pairs.WithWorkDir("/tmp/tmp1"),
+				pairs.WithSize(200),
+				pairs.WithName("def"),
+				pairs.WithSize(300),
+			},
+			nil,
+		},
+		{
+			"not registered pair",
+			"tests://abc/tmp?not_a_pair=a",
+			nil,
+			services.ErrConnectionStringInvalid,
+		},
+		{
+			"key without value is ignored (even not registered pair)",
+			"tests://abc/tmp?not_a_pair&&",
+			[]Pair{
+				pairs.WithName("abc"),
+				pairs.WithWorkDir("/tmp"),
+			},
+			nil,
+		},
+		{
+			"not parseable pair",
+			"tests://abc/tmp?io_call_back=a",
+			nil,
+			services.ErrConnectionStringInvalid,
+		},
+	}
+
+	for _, tt := range cases {
+		t.Run(tt.name, func(t *testing.T) {
+			servicer, err := services.NewServicerFromString(tt.connStr)
+			service, ok := servicer.(*Service)
+
+			if tt.err == nil {
+				assert.Nil(t, err)
+				assert.True(t, ok)
+			} else {
+				assert.True(t, errors.Is(err, tt.err))
+				return
+			}
+
+			assert.Equal(t, service.Pairs, tt.pairs)
+		})
+	}
+}

cmd/definitions/tests/service.go

@@ -10,6 +10,8 @@ type Service struct {
 	defaultPairs DefaultServicePairs
 	features     ServiceFeatures
 
+	Pairs []Pair
+
 	UnimplementedServicer
 }
 

cmd/definitions/tests/service.toml

@@ -74,4 +74,8 @@ description = "is the storage class for this object"
 
 [pairs.size]
 type = "int64"
-description = "tests pair conflict"
+description = "tests pair conflict"
+
+[pairs.string_pair]
+type = "string"
+description = "tests connection string"

cmd/definitions/tests/utils.go

@@ -19,7 +19,7 @@ func (s *Storage) String() string {
 }
 
 func NewServicer(pairs ...typ.Pair) (typ.Servicer, error) {
-	return nil, nil
+	return &Service{Pairs: pairs}, nil
 }
 
 func NewStorager(pairs ...typ.Pair) (typ.Storager, error) {

cmd/definitions/tmpl/service.tmpl

@@ -73,6 +73,13 @@ func With{{ $pname }}(v {{ $v.Type }}) Pair {
 {{- end }}
 {{- end }}
 
+var pairMap = map[string]string {
+    {{- range $_, $v := .Pairs }}
+    {{- $pname := $v.Name | toPascal }}
+    "{{ $v.Name }}": "{{ $v.Type }}",
+    {{- end }}
+}
+
 {{- range $_, $v := .Namespaces }}
     {{- template "interfaces" makeSlice $v.Name $v.Interfaces }}
     {{- template "features" makeSlice $v.Name $v.Funcs }}
@@ -378,4 +385,5 @@ func init() {
     {{- range $_, $v := .Namespaces }}
         services.Register{{ $v.Name | toPascal }}r(Type, New{{ $v.Name | toPascal }}r)
     {{- end }}
+    services.RegisterSchema(Type, pairMap)
 }

services/new.go

@@ -1,8 +1,13 @@
 package services
 
 import (
+	"encoding/base64"
+	"fmt"
+	"strconv"
+	"strings"
 	"sync"
 
+	"github.com/beyondstorage/go-storage/v4/pairs"
 	"github.com/beyondstorage/go-storage/v4/types"
 )
 
@@ -14,27 +19,27 @@ type (
 )
 
 var (
-	serviceFnMap map[string]NewServicerFunc
-	serviceLock  sync.Mutex
+	servicerFnMap map[string]NewServicerFunc
+	servicerLock  sync.Mutex
 
 	storagerFnMap map[string]NewStoragerFunc
 	storagerLock  sync.Mutex
 )
 
 // RegisterServicer will register a servicer.
 func RegisterServicer(ty string, fn NewServicerFunc) {
-	serviceLock.Lock()
-	defer serviceLock.Unlock()
+	servicerLock.Lock()
+	defer servicerLock.Unlock()
 
-	serviceFnMap[ty] = fn
+	servicerFnMap[ty] = fn
 }
 
 // NewServicer will initiate a new servicer.
 func NewServicer(ty string, ps ...types.Pair) (types.Servicer, error) {
-	serviceLock.Lock()
-	defer serviceLock.Unlock()
+	servicerLock.Lock()
+	defer servicerLock.Unlock()
 
-	fn, ok := serviceFnMap[ty]
+	fn, ok := servicerFnMap[ty]
 	if !ok {
 		return nil, InitError{Op: "new_servicer", Type: ty, Err: ErrServiceNotRegistered, Pairs: ps}
 	}
@@ -64,6 +69,150 @@ func NewStorager(ty string, ps ...types.Pair) (types.Storager, error) {
 }
 
 func init() {
-	serviceFnMap = make(map[string]NewServicerFunc)
+	servicerFnMap = make(map[string]NewServicerFunc)
 	storagerFnMap = make(map[string]NewStoragerFunc)
 }
+
+var (
+	servicePairMaps map[string]map[string]string
+	schemaLock      sync.Mutex
+)
+
+// RegisterSchema will register a service's pair map.
+//
+// Users SHOULD NOT call this function.
+func RegisterSchema(ty string, m map[string]string) {
+	schemaLock.Lock()
+	defer schemaLock.Unlock()
+
+	servicePairMaps[ty] = m
+}
+
+func NewServicerFromString(connStr string) (types.Servicer, error) {
+	ty, ps, err := parseConnectionString(connStr)
+	if err != nil {
+		return nil, InitError{Op: "new_servicer", Type: ty, Err: err, Pairs: ps}
+	}
+	return NewServicer(ty, ps...)
+}
+
+func NewStoragerFromString(connStr string) (types.Storager, error) {
+	ty, ps, err := parseConnectionString(connStr)
+	if err != nil {
+		return nil, InitError{Op: "new_storager", Type: ty, Err: err, Pairs: ps}
+	}
+	return NewStorager(ty, ps...)
+}
+
+var (
+	// ErrConnectionStringInvalid means the connection string is invalid.
+	ErrConnectionStringInvalid = NewErrorCode("connection string is invalid")
+)
+
+// <type>://[<name>][<work_dir>][?key1=value1&...&keyN=valueN]
+func parseConnectionString(ConnStr string) (ty string, ps []types.Pair, err error) {
+	colon := strings.Index(ConnStr, ":")
+	if colon == -1 {
+		err = fmt.Errorf("%w: %s, %s", ErrConnectionStringInvalid, "service type missing", ConnStr)
+		return
+	}
+	ty = ConnStr[:colon]
+	rest := ConnStr[colon+1:]
+
+	schemaLock.Lock()
+	m, ok := servicePairMaps[ty]
+	schemaLock.Unlock()
+	if !ok {
+		err = ErrServiceNotRegistered
+		return
+	}
+
+	if !strings.HasPrefix(rest, "//") {
+		err = fmt.Errorf("%w: %s", ErrConnectionStringInvalid, ConnStr)
+		return
+	}
+	rest = rest[2:]
+
+	// [<name>][<work_dir>][?key1=value1&...&keyN=valueN]
+	// <name> does not contain '/'
+	// <work_dir> begins with '/'
+	question := strings.Index(rest, "?")
+	var path string
+	if question == -1 {
+		path = rest
+		rest = ""
+	} else {
+		path = rest[:question]
+		rest = rest[question+1:]
+	}
+
+	if len(path) == 0 {
+		// both <name> and <work_dir> missing
+	} else {
+		slash := strings.Index(path, "/")
+		if slash == -1 {
+			name := path
+			ps = append(ps, pairs.WithName(name))
+		} else if slash == 0 {
+			workDir := path
+			ps = append(ps, pairs.WithWorkDir(workDir))
+		} else {
+			name := path[:slash]
+			workDir := path[slash:]
+			ps = append(ps, pairs.WithName(name), pairs.WithWorkDir(workDir))
+		}
+	}
+
+	for _, v := range strings.Split(rest, "&") {
+		opt := strings.SplitN(v, "=", 2)
+		if len(opt) != 2 {
+			// && or &key&, ignore
+			continue
+		}
+		pair, parseErr := parse(m, opt[0], opt[1])
+		if parseErr != nil {
+			ps = nil
+			err = fmt.Errorf("%w: %v", ErrConnectionStringInvalid, parseErr)
+			return
+		}
+		ps = append(ps, pair)
+	}
+	return
+}
+
+func parse(m map[string]string, k string, v string) (pair types.Pair, err error) {
+	vType, ok := m[k]
+	if !ok {
+		err = fmt.Errorf("pair not registered: %v", k)
+		return types.Pair{}, err
+	}
+
+	pair.Key = k
+
+	switch vType {
+	case "string":
+		pair.Value, err = v, nil
+	case "bool":
+		pair.Value, err = strconv.ParseBool(v)
+	case "int":
+		var i int64
+		i, err = strconv.ParseInt(v, 0, 0)
+		pair.Value = int(i)
+	case "int64":
+		pair.Value, err = strconv.ParseInt(v, 0, 64)
+	case "[]byte":
+		pair.Value, err = base64.RawStdEncoding.DecodeString(v)
+	default:
+		return types.Pair{}, fmt.Errorf("type not parseable: %v, %v", k, vType)
+	}
+
+	if err != nil {
+		pair = types.Pair{}
+		err = fmt.Errorf("pair value invalid: %v, %v, %v: %v", k, vType, v, err)
+	}
+	return
+}
+
+func init() {
+	servicePairMaps = make(map[string]map[string]string)
+}