Skip to content

Commit

Permalink
Merge pull request #11670 from tangcong/optimize-auth-store-log
Browse files Browse the repository at this point in the history
*: optimize auth/etcdserver logs to facilitate troubleshooting data inconsistency
  • Loading branch information
jingyih authored Mar 15, 2020
2 parents 7444d3a + 5a17367 commit 193e60e
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 3 deletions.
10 changes: 7 additions & 3 deletions auth/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -851,8 +851,13 @@ func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeE
if revision == 0 {
return ErrUserEmpty
}

if revision < as.Revision() {
rev := as.Revision()
if revision < rev {
as.lg.Warn("request auth revision is less than current node auth revision",
zap.Uint64("current node auth revision", rev),
zap.Uint64("request auth revision", revision),
zap.ByteString("request key", key),
zap.Error(ErrAuthOldRevision))
return ErrAuthOldRevision
}

Expand Down Expand Up @@ -1065,7 +1070,6 @@ func NewAuthStore(lg *zap.Logger, be backend.Backend, tp TokenProvider, bcryptCo

if as.Revision() == 0 {
as.commitRevision(tx)
as.saveConsistentIndex(tx)
}

as.setupMetricsReporter()
Expand Down
3 changes: 3 additions & 0 deletions etcdserver/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
ar := &applyResult{}
defer func(start time.Time) {
warnOfExpensiveRequest(a.s.getLogger(), start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
if ar.err != nil {
warnOfFailedRequest(a.s.getLogger(), start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
}
}(time.Now())

// call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls
Expand Down
15 changes: 15 additions & 0 deletions etcdserver/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,21 @@ func warnOfExpensiveRequest(lg *zap.Logger, now time.Time, reqStringer fmt.Strin
warnOfExpensiveGenericRequest(lg, now, reqStringer, "", resp, err)
}

func warnOfFailedRequest(lg *zap.Logger, now time.Time, reqStringer fmt.Stringer, respMsg proto.Message, err error) {
var resp string
if !isNil(respMsg) {
resp = fmt.Sprintf("size:%d", proto.Size(respMsg))
}
d := time.Since(now)
lg.Warn(
"failed to apply request",
zap.Duration("took", d),
zap.String("request", reqStringer.String()),
zap.String("response", resp),
zap.Error(err),
)
}

func warnOfExpensiveReadOnlyTxnRequest(lg *zap.Logger, now time.Time, r *pb.TxnRequest, txnResponse *pb.TxnResponse, err error) {
reqStringer := pb.NewLoggableTxnRequest(r)
var resp string
Expand Down

0 comments on commit 193e60e

Please sign in to comment.