diff --git a/go.mod b/go.mod index 50d23c30..4c69fa56 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/go-chassis/go-archaius require ( github.com/Shonminh/apollo-client v0.2.0 - github.com/apache/servicecomb-kie v0.1.1-0.20200303014812-959a30975b44 github.com/fsnotify/fsnotify v1.4.7 github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4 github.com/go-mesh/openlogging v1.0.1 diff --git a/pkg/kieclient/kieclient.go b/pkg/kieclient/kieclient.go new file mode 100644 index 00000000..5dad87d9 --- /dev/null +++ b/pkg/kieclient/kieclient.go @@ -0,0 +1,308 @@ +/* + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kieclient + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/go-chassis/foundation/httpclient" + "github.com/go-chassis/foundation/security" + "github.com/go-mesh/openlogging" +) + +//match mode +const ( + QueryParamQ = "q" + QueryByLabelsCon = "&" + QueryParamMatch = "match" + QueryParamKeyID = "kv_id" +) + +//http headers +const ( + HeaderRevision = "X-Kie-Revision" + HeaderContentType = "Content-Type" +) + +//ContentType +const ( + ContentTypeJSON = "application/json" +) + +//const +const ( + version = "v1" + APIPathKV = "kie/kv" + + MsgGetFailed = "get failed" + FmtGetFailed = "get %s failed,http status [%s], body [%s]" +) + +//client errors +var ( + ErrKeyNotExist = errors.New("can not find value") + ErrNoChanges = errors.New("kv has not been changed since last polling") +) + +//Client is the servicecomb kie rest client. +//it is concurrency safe +type Client struct { + opts Config + cipher security.Cipher + c *httpclient.Requests + currentRevision int +} + +//Config is the config of client +type Config struct { + Endpoint string + DefaultLabels map[string]string + VerifyPeer bool //TODO make it works, now just keep it false +} + +//New create a client +func New(config Config) (*Client, error) { + u, err := url.Parse(config.Endpoint) + if err != nil { + return nil, err + } + httpOpts := &httpclient.Options{} + if u.Scheme == "https" { + // #nosec + httpOpts.TLSConfig = &tls.Config{ + InsecureSkipVerify: !config.VerifyPeer, + } + } + c, err := httpclient.New(httpOpts) + if err != nil { + return nil, err + } + return &Client{ + opts: config, + c: c, + }, nil +} + +//Put create value of a key +func (c *Client) Put(ctx context.Context, kv KVRequest, opts ...OpOption) (*KVDoc, error) { + options := OpOptions{} + for _, o := range opts { + o(&options) + } + if options.Project == "" { + options.Project = defaultProject + } + url := fmt.Sprintf("%s/%s/%s/%s/%s", c.opts.Endpoint, version, options.Project, APIPathKV, kv.Key) + h := http.Header{} + h.Set(HeaderContentType, ContentTypeJSON) + body, _ := json.Marshal(kv) + resp, err := c.c.Do(ctx, http.MethodPut, url, h, body) + if err != nil { + return nil, err + } + b := ReadBody(resp) + if resp.StatusCode != http.StatusOK { + openlogging.Error(MsgGetFailed, openlogging.WithTags(openlogging.Tags{ + "k": kv.Key, + "status": resp.Status, + "body": b, + })) + return nil, fmt.Errorf(FmtGetFailed, kv.Key, resp.Status, b) + } + + kvs := &KVDoc{} + err = json.Unmarshal(b, kvs) + if err != nil { + openlogging.Error("unmarshal kv failed:" + err.Error()) + return nil, err + } + return kvs, nil +} + +//Get get value of a key +func (c *Client) Get(ctx context.Context, opts ...GetOption) (*KVResponse, int, error) { + options := GetOptions{} + for _, o := range opts { + o(&options) + } + if options.Project == "" { + options.Project = defaultProject + } + if options.Revision == "" { + options.Revision = strconv.Itoa(c.currentRevision) + } + var url string + if options.Key != "" { + url = fmt.Sprintf("%s/%s/%s/%s/%s?revision=%s", c.opts.Endpoint, version, options.Project, APIPathKV, options.Key, options.Revision) + } else { + url = fmt.Sprintf("%s/%s/%s/%s?revision=%s", c.opts.Endpoint, version, options.Project, APIPathKV, options.Revision) + } + if options.Wait != "" { + url = url + "&wait=" + options.Wait + } + if options.Exact { + url = url + "&" + QueryParamMatch + "=exact" + } + labels := "" + if len(options.Labels) != 0 { + for k, v := range options.Labels[0] { + labels = labels + "&label=" + k + ":" + v + } + url = url + labels + } + h := http.Header{} + resp, err := c.c.Do(ctx, http.MethodGet, url, h, nil) + if err != nil { + return nil, -1, err + } + responseRevision, err := strconv.Atoi(resp.Header.Get(HeaderRevision)) + if err != nil { + responseRevision = -1 + } + b := ReadBody(resp) + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, responseRevision, ErrKeyNotExist + } + if resp.StatusCode == http.StatusNotModified { + return nil, responseRevision, ErrNoChanges + } + openlogging.Error(MsgGetFailed, openlogging.WithTags(openlogging.Tags{ + "k": options.Key, + "status": resp.Status, + "body": b, + })) + return nil, responseRevision, fmt.Errorf(FmtGetFailed, options.Key, resp.Status, b) + } else if err != nil { + msg := fmt.Sprintf("get revision from response header failed when the request status is OK: %v", err) + openlogging.Error(msg) + return nil, responseRevision, fmt.Errorf(msg) + } + var kvs *KVResponse + err = json.Unmarshal(b, &kvs) + if err != nil { + openlogging.Error("unmarshal kv failed:" + err.Error()) + return nil, responseRevision, err + } + c.currentRevision = responseRevision + return kvs, responseRevision, nil +} + +//Summary get value by labels +func (c *Client) Summary(ctx context.Context, opts ...GetOption) ([]*KVResponse, error) { + options := GetOptions{} + for _, o := range opts { + o(&options) + } + if options.Project == "" { + options.Project = defaultProject + } + labelParams := "" + for _, labels := range options.Labels { + labelParams += QueryParamQ + "=" + for labelKey, labelValue := range labels { + labelParams += labelKey + ":" + labelValue + "+" + } + if labels != nil && len(labels) > 0 { + labelParams = strings.TrimRight(labelParams, "+") + } + labelParams += QueryByLabelsCon + } + if options.Labels != nil && len(options.Labels) > 0 { + labelParams = strings.TrimRight(labelParams, QueryByLabelsCon) + } + url := fmt.Sprintf("%s/%s/%s/%s?%s", c.opts.Endpoint, version, options.Project, "kie/summary", labelParams) + h := http.Header{} + resp, err := c.c.Do(ctx, http.MethodGet, url, h, nil) + if err != nil { + return nil, err + } + b := ReadBody(resp) + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return nil, ErrKeyNotExist + } + openlogging.Error(MsgGetFailed, openlogging.WithTags(openlogging.Tags{ + "p": options.Project, + "status": resp.Status, + "body": b, + })) + return nil, fmt.Errorf("search %s failed,http status [%s], body [%s]", labelParams, resp.Status, b) + } + var kvs []*KVResponse + err = json.Unmarshal(b, &kvs) + if err != nil { + openlogging.Error("unmarshal kv failed:" + err.Error()) + return nil, err + } + return kvs, nil +} + +//Delete remove kv +func (c *Client) Delete(ctx context.Context, kvID, labelID string, opts ...OpOption) error { + options := OpOptions{} + for _, o := range opts { + o(&options) + } + if options.Project == "" { + options.Project = defaultProject + } + url := fmt.Sprintf("%s/%s/%s/%s/?%s=%s", c.opts.Endpoint, version, options.Project, APIPathKV, + QueryParamKeyID, kvID) + if labelID != "" { + url = fmt.Sprintf("%s?labelID=%s", url, labelID) + } + h := http.Header{} + h.Set(HeaderContentType, ContentTypeJSON) + resp, err := c.c.Do(ctx, http.MethodDelete, url, h, nil) + if err != nil { + return err + } + b := ReadBody(resp) + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("delete %s failed,http status [%s], body [%s]", kvID, resp.Status, b) + } + return nil +} + +//CurrentRevision return the current revision of kie, which is updated on the last get request +func (c *Client) CurrentRevision() int { + return c.currentRevision +} + +// ReadBody read body from the from the response +func ReadBody(resp *http.Response) []byte { + if resp != nil && resp.Body != nil { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + openlogging.Error(fmt.Sprintf("read body failed: %s", err.Error())) + return nil + } + return body + } + openlogging.Error("response body or response is nil") + return nil +} diff --git a/pkg/kieclient/options.go b/pkg/kieclient/options.go new file mode 100644 index 00000000..09b6c85a --- /dev/null +++ b/pkg/kieclient/options.go @@ -0,0 +1,95 @@ +/* + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kieclient + +import "strconv" + +const ( + defaultProject = "default" +) + +//GetOption is the functional option of client func +type GetOption func(*GetOptions) + +//OpOption is the functional option of client func +type OpOption func(*OpOptions) + +//GetOptions is the options of client func +type GetOptions struct { + Labels []map[string]string + Project string + Key string + Wait string + Exact bool + Revision string +} + +//OpOptions is the options of client func +type OpOptions struct { + Project string +} + +//WithLabels query kv by labels +func WithLabels(l ...map[string]string) GetOption { + return func(options *GetOptions) { + for _, labels := range l { + options.Labels = append(options.Labels, labels) + } + } +} + +//WithGetProject query keys with certain project +func WithGetProject(project string) GetOption { + return func(options *GetOptions) { + options.Project = project + } +} + +//WithExact means label exact match +func WithExact() GetOption { + return func(options *GetOptions) { + options.Exact = true + } +} + +//WithWait is for long polling,format is 1s,2m +func WithWait(duration string) GetOption { + return func(options *GetOptions) { + options.Wait = duration + } +} + +//WithKey query keys with certain key +func WithKey(k string) GetOption { + return func(options *GetOptions) { + options.Key = k + } +} + +//WithRevision query keys with certain revision +func WithRevision(revision int) GetOption { + return func(options *GetOptions) { + options.Revision = strconv.Itoa(revision) + } +} + +//WithProject set project to param +func WithProject(project string) OpOption { + return func(options *OpOptions) { + options.Project = project + } +} diff --git a/pkg/kieclient/struct.go b/pkg/kieclient/struct.go new file mode 100644 index 00000000..a53af363 --- /dev/null +++ b/pkg/kieclient/struct.go @@ -0,0 +1,59 @@ +/* + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kieclient + +//KVRequest is http request body +type KVRequest struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value,omitempty" yaml:"value,omitempty"` + ValueType string `json:"value_type,omitempty" bson:"value_type,omitempty" yaml:"value_type,omitempty"` //ini,json,text,yaml,properties + Checker string `json:"check,omitempty" yaml:"check,omitempty"` //python script + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` //redundant +} + +//KVResponse represents the key value list +type KVResponse struct { + LabelDoc *LabelDocResponse `json:"label,omitempty"` + Total int `json:"total,omitempty"` + Data []*KVDoc `json:"data,omitempty"` +} + +//LabelDocResponse is label struct +type LabelDocResponse struct { + LabelID string `json:"label_id,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +//KVDoc is database struct to store kv +type KVDoc struct { + ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string"` + LabelID string `json:"label_id,omitempty" bson:"label_id,omitempty" yaml:"label_id,omitempty"` + Key string `json:"key" yaml:"key"` + Value string `json:"value,omitempty" yaml:"value,omitempty"` + ValueType string `json:"value_type,omitempty" bson:"value_type,omitempty" yaml:"value_type,omitempty"` //ini,json,text,yaml,properties + Checker string `json:"check,omitempty" yaml:"check,omitempty"` //python script + CreateRevision int64 `json:"create_revision,omitempty" bson:"create_revision," yaml:"create_revision,omitempty"` + UpdateRevision int64 `json:"update_revision,omitempty" bson:"update_revision," yaml:"update_revision,omitempty"` + Project string `json:"project,omitempty" yaml:"project,omitempty"` + Status string `json:"status,omitempty" yaml:"status,omitempty"` + CreatTime string `json:"create_time,omitempty" yaml:"create_time,omitempty"` + UpdateTime string `json:"update_time,omitempty" yaml:"update_time,omitempty"` + + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` //redundant + Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` //redundant + +} diff --git a/source/remote/kie/kie_client.go b/source/remote/kie/kie_client.go index 38a8c70b..e5ca6774 100644 --- a/source/remote/kie/kie_client.go +++ b/source/remote/kie/kie_client.go @@ -23,8 +23,7 @@ import ( "sync" "time" - "github.com/apache/servicecomb-kie/client" - "github.com/apache/servicecomb-kie/pkg/model" + client "github.com/go-chassis/go-archaius/pkg/kieclient" "github.com/go-chassis/go-archaius/source/remote" "github.com/go-chassis/go-archaius/source/util/queue" "github.com/go-mesh/openlogging" @@ -205,7 +204,7 @@ func (k *Kie) watchKVDimensionally(f func(map[string]interface{}), errHandler fu } } -func (k *Kie) setDimensionConfigs(kv *model.KVResponse, dimension DimensionName) bool { +func (k *Kie) setDimensionConfigs(kv *client.KVResponse, dimension DimensionName) bool { if k.dimensions[dimension] == nil { return false } diff --git a/source/remote/kie/kie_client_test.go b/source/remote/kie/kie_client_test.go index 85db6159..880eb628 100644 --- a/source/remote/kie/kie_client_test.go +++ b/source/remote/kie/kie_client_test.go @@ -4,7 +4,7 @@ import ( "strconv" "testing" - "github.com/apache/servicecomb-kie/pkg/model" + client "github.com/go-chassis/go-archaius/pkg/kieclient" "github.com/go-chassis/go-archaius/source/remote" "github.com/stretchr/testify/assert" ) @@ -27,8 +27,8 @@ func TestMergeConfig(t *testing.T) { remote.LabelVersion: "1.0.0", }}) for i, dimension := range dimensionPrecedence { - k.setDimensionConfigs(&model.KVResponse{ - Data: []*model.KVDoc{ + k.setDimensionConfigs(&client.KVResponse{ + Data: []*client.KVDoc{ { Key: "foo", Value: strconv.Itoa(i + 1),