From ade7875f25897643a5fe465dcdbd8f5324ef80e3 Mon Sep 17 00:00:00 2001 From: zyl <2025320615@qq.com> Date: Sat, 26 Dec 2020 16:24:36 +0800 Subject: [PATCH] =?UTF-8?q?#79=20=E6=B6=88=E6=81=AF=E8=BD=A8=E8=BF=B9?= =?UTF-8?q?=E6=8E=A8=E9=80=81=E8=B7=9F=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deployments/db.js | 29 ++++++++-------- go.sum | 4 +++ pkg/common/common.go | 34 +++++++++---------- pkg/model/db_schema.go | 24 +++++++------ server/handler/track_handler.go | 5 ++- server/resource/v1/common.go | 1 + server/resource/v1/history_resource.go | 8 +++++ server/service/mongo/session/session.go | 11 ++---- .../service/mongo/track/polling_detail_dao.go | 6 +++- 9 files changed, 70 insertions(+), 52 deletions(-) diff --git a/deployments/db.js b/deployments/db.js index 42b919fb..99f2c710 100644 --- a/deployments/db.js +++ b/deployments/db.js @@ -83,7 +83,7 @@ db.createCollection( "polling_detail", { max: 100, validator: { $jsonSchema: { bsonType: "object", - required: [ "id","session_id","domain","url_path" ], + required: [ "id","session_id","revision","url_path" ], properties: { id: { bsonType: "string", @@ -91,7 +91,7 @@ db.createCollection( "polling_detail", { domain: { bsonType: "string", }, - params: { + polling_data: { bsonType: "object" }, ip: { @@ -103,8 +103,8 @@ db.createCollection( "polling_detail", { response_body: { bsonType: "object" }, - response_header: { - bsonType: "object" + revision: { + bsonType: "string" }, response_code: { bsonType: "number" @@ -114,15 +114,14 @@ db.createCollection( "polling_detail", { } ); //index -db.kv.createIndex({"id": 1}, { unique: true } ); -db.kv.createIndex({key: 1, label_format: 1,domain:1,project:1},{ unique: true }); -db.kv_revision.createIndex( { "delete_time": 1 }, { expireAfterSeconds: 7 * 24 * 3600 } ); -db.label.createIndex({"id": 1}, { unique: true } ); -db.label.createIndex({format: 1,domain:1,project:1},{ unique: true }); -db.polling_detail.createIndex({"id": 1}, { unique: true } ); -db.polling_detail.createIndex({session_id:1,domain:1}, { unique: true } ); -db.counter.createIndex({name: 1,domain:1},{ unique: true }); -db.view.createIndex({"id": 1}, { unique: true } ); -db.view.createIndex({display:1,domain:1,project:1},{ unique: true }); +db.kv.createIndex({"id": 1}, {unique: true}); +db.kv.createIndex({key: 1, label_format: 1, domain: 1, project: 1}, {unique: true}); +db.kv_revision.createIndex({"delete_time": 1}, {expireAfterSeconds: 7 * 24 * 3600}); +db.label.createIndex({"id": 1}, {unique: true}); +db.label.createIndex({format: 1, domain: 1, project: 1}, {unique: true}); +db.polling_detail.createIndex({timestamp: 1}, {expireAfterSeconds: 7 * 24 * 3600}); +db.counter.createIndex({name: 1, domain: 1}, {unique: true}); +db.view.createIndex({"id": 1}, {unique: true}); +db.view.createIndex({display: 1, domain: 1, project: 1}, {unique: true}); //db config -db.setProfilingLevel(1, {slowms: 80, sampleRate: 1} ); \ No newline at end of file +db.setProfilingLevel(1, {slowms: 80, sampleRate: 1}); \ No newline at end of file diff --git a/go.sum b/go.sum index 309b9e1c..6d39ae31 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4 h1:wx8JXvg github.com/go-chassis/foundation v0.1.1-0.20191113114104-2b05871e9ec4/go.mod h1:21/ajGtgJlWTCeM0TxGJdRhO8bJkKirWyV8Stlh6g6c= github.com/go-chassis/foundation v0.1.1-0.20200825060850-b16bf420f7b3 h1:c+bwT0qLY69jSU8TmzuNcb9UL/QFAiU96kjuX5TMiQc= github.com/go-chassis/foundation v0.1.1-0.20200825060850-b16bf420f7b3/go.mod h1:21/ajGtgJlWTCeM0TxGJdRhO8bJkKirWyV8Stlh6g6c= +github.com/go-chassis/foundation v0.2.2-0.20201208060254-d5e8e5beb1f0 h1:6oazFyZKryG3r7K20UuWx+bclHMFGXjKBSMJpCefmMI= github.com/go-chassis/foundation v0.2.2-0.20201208060254-d5e8e5beb1f0/go.mod h1:2PjwqpVwYEVaAldl5A58a08viH8p27pNeYaiE3ZxOBA= github.com/go-chassis/go-archaius v1.2.1-0.20200309104817-8c3d4e87d33c h1:pimEM4Oy/Uf4xG4G7TrRUQbIRFAfHiarxDQQS2gmKaM= github.com/go-chassis/go-archaius v1.2.1-0.20200309104817-8c3d4e87d33c/go.mod h1:gVP52u/jCU0fgUjXdUW1VLp5YLLJ+Yl2zoOPrLM/WOM= @@ -134,6 +135,7 @@ github.com/go-chassis/go-chassis/v2 v2.0.2 h1:FMiZgAa6IjDobtZN5oql+eSgMIlDq5+12o github.com/go-chassis/go-chassis/v2 v2.0.2/go.mod h1:xgXXrRUgn5sybtecTDn60ypepHKIV9QpSqR3CIEBNt0= github.com/go-chassis/go-chassis/v2 v2.1.0 h1:NnVUBj3U/pPijpZGooH+3QIRlOB2nAuZtx9b71aiaig= github.com/go-chassis/go-chassis/v2 v2.1.0/go.mod h1:DHA83QKVmuOtdkaWCP5yCDOh+GIRH+YpX3uZ2BlGMWw= +github.com/go-chassis/go-chassis/v2 v2.1.1-0.20201208062518-9c2e86bd7a6c h1:k+D2w1oZ/fQjingJc1KzuwGv0cJKfaGDgaOW/4T4Q1c= github.com/go-chassis/go-chassis/v2 v2.1.1-0.20201208062518-9c2e86bd7a6c/go.mod h1:C5j2kuBAdch4DsSjKQxiYJr3Xy0PoZu2Bxx9QIJIArI= github.com/go-chassis/go-restful-swagger20 v1.0.3-0.20200310030431-17d80f34264f h1:5QmmNpVcGqIc6tuKNe5EAI4PA8Yn2EL9Oee7YdcJ4PE= github.com/go-chassis/go-restful-swagger20 v1.0.3-0.20200310030431-17d80f34264f/go.mod h1:eW62fYuzlNFDvIacB6AV8bhUDCTy4myfTCv0bT9Gbb0= @@ -432,6 +434,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -598,6 +601,7 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/common/common.go b/pkg/common/common.go index 097a2b1c..2ed8a39c 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -21,23 +21,23 @@ import "time" //match mode const ( - QueryParamQ = "q" - QueryByLabelsCon = "&" - QueryParamWait = "wait" - QueryParamRev = "revision" - QueryParamMatch = "match" - QueryParamKey = "key" - QueryParamLabel = "label" - QueryParamStatus = "status" - QueryParamOffset = "offset" - QueryParamLimit = "limit" - PathParamKVID = "kv_id" - PathParameterProject = "project" - //polling data - QueryParamSessionID = "sessionId" - QueryParamIP = "ip" - QueryParamURLPath = "urlPath" - QueryParamUserAgent = "userAgent" + QueryParamQ = "q" + QueryByLabelsCon = "&" + QueryParamWait = "wait" + QueryParamRev = "revision" + QueryParamMatch = "match" + QueryParamKey = "key" + QueryParamLabel = "label" + QueryParamStatus = "status" + QueryParamOffset = "offset" + QueryParamLimit = "limit" + PathParamKVID = "kv_id" + PathParameterProject = "project" + QueryParamSessionID = "sessionId" + QueryParamSessionGroup = "sessionGroup" + QueryParamIP = "ip" + QueryParamURLPath = "urlPath" + QueryParamUserAgent = "userAgent" ) //http headers diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go index 4d6fe506..8875c242 100644 --- a/pkg/model/db_schema.go +++ b/pkg/model/db_schema.go @@ -17,6 +17,8 @@ package model +import "time" + //LabelDoc is database struct to store labels type LabelDoc struct { ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string"` @@ -57,16 +59,18 @@ type ViewDoc struct { //PollingDetail is db struct, it record operation history type PollingDetail struct { - ID string `json:"id,omitempty" yaml:"id,omitempty"` - SessionID string `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"` - Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` - PollingData map[string]interface{} `json:"polling_data,omitempty" yaml:"polling_data,omitempty"` - IP string `json:"ip,omitempty" yaml:"ip,omitempty"` - UserAgent string `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"` - URLPath string `json:"url_path,omitempty" bson:"url_path," yaml:"url_path,omitempty"` - ResponseBody []*KVDoc `json:"kv,omitempty" bson:"kv," yaml:"kv,omitempty"` - ResponseHeader map[string][]string `json:"response_header,omitempty" bson:"response_header," yaml:"response_header,omitempty"` - ResponseCode int `json:"response_code,omitempty" bson:"response_code," yaml:"response_code,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + SessionID string `json:"session_id,omitempty" bson:"session_id," yaml:"session_id,omitempty"` + SessionGroup string `json:"session_group,omitempty" bson:"session_group," yaml:"session_group,omitempty"` + Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` + PollingData map[string]interface{} `json:"polling_data,omitempty" yaml:"polling_data,omitempty"` + Revision string `json:"revision,omitempty" yaml:"revision,omitempty"` + IP string `json:"ip,omitempty" yaml:"ip,omitempty"` + UserAgent string `json:"user_agent,omitempty" bson:"user_agent," yaml:"user_agent,omitempty"` + URLPath string `json:"url_path,omitempty" bson:"url_path," yaml:"url_path,omitempty"` + ResponseBody []*KVDoc `json:"kv,omitempty" bson:"kv," yaml:"kv,omitempty"` + ResponseCode int `json:"response_code,omitempty" bson:"response_code," yaml:"response_code,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty" yaml:"timestamp,omitempty"` } // UpdateKVRequest is db struct, it contains kv update request params diff --git a/server/handler/track_handler.go b/server/handler/track_handler.go index bddd7a8b..4228c984 100644 --- a/server/handler/track_handler.go +++ b/server/handler/track_handler.go @@ -29,6 +29,7 @@ import ( "github.com/go-chassis/openlog" "net/http" "strings" + "time" ) //const of noop auth handler @@ -66,13 +67,15 @@ func (h *TrackHandler) Handle(chain *handler.Chain, inv *invocation.Invocation, data := &model.PollingDetail{} data.URLPath = req.Request.Method + " " + req.Request.URL.Path data.SessionID = sessionID + data.SessionGroup = req.HeaderParameter(v1.HeaderSessionGroup) data.UserAgent = req.HeaderParameter(v1.HeaderUserAgent) data.Domain = v1.ReadDomain(req.Request.Context()) data.IP = iputil.ClientIP(req.Request) data.ResponseBody = req.Attribute(common.RespBodyContextKey).([]*model.KVDoc) data.ResponseCode = ir.Status + data.Timestamp = time.Now() if resp != nil { - data.ResponseHeader = resp.Header() + data.Revision = resp.Header().Get(common.HeaderRevision) } data.PollingData = map[string]interface{}{ "revision": revStr, diff --git a/server/resource/v1/common.go b/server/resource/v1/common.go index 21b80c1e..8500e415 100644 --- a/server/resource/v1/common.go +++ b/server/resource/v1/common.go @@ -43,6 +43,7 @@ import ( const ( HeaderUserAgent = "User-Agent" HeaderSessionID = "X-Session-Id" + HeaderSessionGroup = "X-Session-Group" AttributeDomainKey = "domain" FmtReadRequestError = "decode request body failed: %v" diff --git a/server/resource/v1/history_resource.go b/server/resource/v1/history_resource.go index ed80d978..acf7fb1b 100644 --- a/server/resource/v1/history_resource.go +++ b/server/resource/v1/history_resource.go @@ -72,6 +72,14 @@ func (r *HistoryResource) GetPollingData(context *restful.Context) { if sessionID != "" { query.SessionID = sessionID } + sessionCroup := context.ReadQueryParameter(common.QueryParamSessionGroup) + if sessionCroup != "" { + query.SessionGroup = sessionCroup + } + revision := context.ReadQueryParameter(common.QueryParamRev) + if revision != "" { + query.Revision = revision + } ip := context.ReadQueryParameter(common.QueryParamIP) if ip != "" { query.IP = ip diff --git a/server/service/mongo/session/session.go b/server/service/mongo/session/session.go index 16e3d6b2..e55e16b7 100644 --- a/server/service/mongo/session/session.go +++ b/server/service/mongo/session/session.go @@ -222,19 +222,14 @@ func ensureKVLongPolling(session *mgo.Session) { c := session.DB(DBName).C(CollectionPollingDetail) err := c.Create(&mgo.CollectionInfo{Validator: bson.M{ "id": bson.M{"$exists": true}, - "params": bson.M{"$exists": true}, + "revision": bson.M{"$exists": true}, "session_id": bson.M{"$exists": true}, "url_path": bson.M{"$exists": true}, }}) wrapError(err, MsgDBExists) err = c.EnsureIndex(mgo.Index{ - Key: []string{"id"}, - Unique: true, - }) - wrapError(err) - err = c.EnsureIndex(mgo.Index{ - Key: []string{"session_id", "domain"}, - Unique: true, + Key: []string{"timestamp"}, + ExpireAfter: 7 * 24 * time.Hour, }) wrapError(err) } diff --git a/server/service/mongo/track/polling_detail_dao.go b/server/service/mongo/track/polling_detail_dao.go index bb3bcae6..f9cc8c5b 100644 --- a/server/service/mongo/track/polling_detail_dao.go +++ b/server/service/mongo/track/polling_detail_dao.go @@ -29,9 +29,10 @@ import ( ) //CreateOrUpdate create a record or update exist record +//If revision and session_id is exist: update else:insert func CreateOrUpdate(ctx context.Context, detail *model.PollingDetail) (*model.PollingDetail, error) { collection := session.GetDB().Collection(session.CollectionPollingDetail) - queryFilter := bson.M{"domain": detail.Domain, "session_id": detail.SessionID} + queryFilter := bson.M{"revision": detail.Domain, "session_id": detail.SessionID} res := collection.FindOne(ctx, queryFilter) if res.Err() != nil { if res.Err() == mongo.ErrNoDocuments { @@ -67,6 +68,9 @@ func Get(ctx context.Context, detail *model.PollingDetail) ([]*model.PollingDeta if detail.URLPath != "" { queryFilter["url_path"] = detail.URLPath } + if detail.Revision != "" { + queryFilter["revision"] = detail.Revision + } cur, err := collection.Find(ctx, queryFilter) if err != nil { return nil, err