From ff2f337c18b0e5adee15540bc833109892fb1295 Mon Sep 17 00:00:00 2001 From: sumeerbhola Date: Tue, 15 Dec 2020 09:06:21 -0500 Subject: [PATCH] [WIP] concurrency: move finalizedTxnCache into lock table (There are bugs here, since existing tests fail. And I have not added new tests yet -- I would like an opinion on the high-level approach before fixing those.) This is a cleanup in preparation for the future, and also has some, probably minor, immediate benefits. In the future, the lock table will support multiple intents for the same key if all but one are known to be finalized. So the finalizedTxnCache belongs in the lock table data-structure. Additionally, we will support intent resolution without holding latches, which has some implications on data-structure consistency: request evaluation will not be allowed to add discovered intents to the lock table since the discovery may be stale. This PR is not changing this discovery behavior since we need it for now (due to interleaved intents), but it moves us along the path towards the lock table data-structure not relying on external behavior for maintaining its in-memory "cache" of locks. Specifically, removing intents from the lock table when the intent is still present in the engine is not principled. We currently do this in two places: - for optimizing limited scans: a later PR will fix this properly by checking the lock table after request evaluation, as outlined in #49973. - using the finalizedTxnCache in the lockTableWaiterImpl: this use is changed in this PR. The code in the lock table also does removal of intents before resolution, but there is a TODO to fix that in the future. It should be easier to do this with the behavior contained in the lock table. The immediate benefits, which may not have any practical significance, are: - We no longer resolve unreplicated locks -- they are simply removed. - A replicated lock is removed from the lock table data-structure only when the requester has finished a scan and is in a position to do resolution. Earlier one could remove the lock but block on another lock, and not do intent resolution on the first lock. This would cause wasteful evaluation of other requests. Release note: None --- .../concurrency/concurrency_control.go | 16 +- .../concurrency/concurrency_manager.go | 11 +- pkg/kv/kvserver/concurrency/lock_table.go | 154 ++++++- .../kvserver/concurrency/lock_table_waiter.go | 79 +--- .../concurrency/lock_table_waiter_test.go | 8 +- .../clear_abandoned_intents | 418 ++++++++++++++++++ 6 files changed, 591 insertions(+), 95 deletions(-) diff --git a/pkg/kv/kvserver/concurrency/concurrency_control.go b/pkg/kv/kvserver/concurrency/concurrency_control.go index de7e90222cd8..acbf453f7492 100644 --- a/pkg/kv/kvserver/concurrency/concurrency_control.go +++ b/pkg/kv/kvserver/concurrency/concurrency_control.go @@ -567,6 +567,12 @@ type lockTable interface { // txn.WriteTimestamp. UpdateLocks(*roachpb.LockUpdate) error + // Informs the lock table that a transaction is finalized. This is used + // by the lock table in a best-effort manner to avoid waiting on locks + // of finalized transactions and telling the caller via + // lockTableGuard.ResolveBeforeEvaluation to resolve a batch of intents. + TransactionIsFinalized(*roachpb.Transaction) + // String returns a debug string representing the state of the lockTable. String() string } @@ -588,6 +594,11 @@ type lockTableGuard interface { // CurState returns the latest waiting state. CurState() waitingState + + // ResolveBeforeScanning lists the locks to resolve before scanning again. + // This must be called after the waiting state has transitioned to + // doneWaiting. + ResolveBeforeScanning() []roachpb.LockUpdate } // lockTableWaiter is concerned with waiting in lock wait-queues for locks held @@ -646,11 +657,6 @@ type lockTableWaiter interface { // and, in turn, remove this method. This will likely fall out of pulling // all replicated locks into the lockTable. WaitOnLock(context.Context, Request, *roachpb.Intent) *Error - - // ClearCaches wipes all caches maintained by the lockTableWaiter. This is - // primarily used to recover memory when a replica loses a lease. However, - // it is also used in tests to reset the state of the lockTableWaiter. - ClearCaches() } // txnWaitQueue holds a collection of wait-queues for transaction records. diff --git a/pkg/kv/kvserver/concurrency/concurrency_manager.go b/pkg/kv/kvserver/concurrency/concurrency_manager.go index 2e5e53a8d5d4..f7ea31a1ca81 100644 --- a/pkg/kv/kvserver/concurrency/concurrency_manager.go +++ b/pkg/kv/kvserver/concurrency/concurrency_manager.go @@ -72,6 +72,9 @@ func (c *Config) initDefaults() { func NewManager(cfg Config) Manager { cfg.initDefaults() m := new(managerImpl) + lt := &lockTableImpl{ + maxLocks: cfg.MaxLockTableSize, + } *m = managerImpl{ // TODO(nvanbenschoten): move pkg/storage/spanlatch to a new // pkg/storage/concurrency/latch package. Make it implement the @@ -82,14 +85,13 @@ func NewManager(cfg Config) Manager { cfg.SlowLatchGauge, ), }, - lt: &lockTableImpl{ - maxLocks: cfg.MaxLockTableSize, - }, + lt: lt, ltw: &lockTableWaiterImpl{ st: cfg.Settings, stopper: cfg.Stopper, ir: cfg.IntentResolver, lm: m, + lt: lt, disableTxnPushing: cfg.DisableTxnPushing, }, // TODO(nvanbenschoten): move pkg/storage/txnwait to a new @@ -344,9 +346,6 @@ func (m *managerImpl) OnRangeLeaseUpdated(seq roachpb.LeaseSequence, isLeasehold const disable = true m.lt.Clear(disable) m.twq.Clear(disable) - // Also clear caches, since they won't be needed any time soon and - // consume memory. - m.ltw.ClearCaches() } } diff --git a/pkg/kv/kvserver/concurrency/lock_table.go b/pkg/kv/kvserver/concurrency/lock_table.go index 554268f83d53..cdafb4907cc3 100644 --- a/pkg/kv/kvserver/concurrency/lock_table.go +++ b/pkg/kv/kvserver/concurrency/lock_table.go @@ -185,6 +185,17 @@ type lockTableImpl struct { locks [spanset.NumSpanScope]treeMu maxLocks int64 + + // finalizedTxnCache is a small LRU cache that tracks transactions that + // were pushed and found to be finalized (COMMITTED or ABORTED). It is + // used as an optimization to avoid repeatedly pushing the transaction + // record when cleaning up the intents of an abandoned transaction. + // + // NOTE: it probably makes sense to maintain a single finalizedTxnCache + // across all Ranges on a Store instead of an individual cache per + // Range. For now, we don't do this because we don't share any state + // between separate concurrency.Manager instances. + finalizedTxnCache txnCache } var _ lockTable = &lockTableImpl{} @@ -256,6 +267,7 @@ var _ lockTable = &lockTableImpl{} // lockTableGuard that returns false from StartWaiting()). type lockTableGuardImpl struct { seqNum uint64 + lt *lockTableImpl // Information about this request. txn *enginepb.TxnMeta @@ -332,6 +344,10 @@ type lockTableGuardImpl struct { // (proportional to number of waiters). mustFindNextLockAfter bool } + // Locks to resolve before scanning again. Doesn't need to be protected by + // mu since should only be read after the caller has already synced with mu + // in realizing that it is doneWaiting. + toResolve []roachpb.LockUpdate } var _ lockTableGuard = &lockTableGuardImpl{} @@ -379,6 +395,10 @@ func (g *lockTableGuardImpl) ShouldWait() bool { return g.mu.startWait } +func (g *lockTableGuardImpl) ResolveBeforeScanning() []roachpb.LockUpdate { + return g.toResolve +} + func (g *lockTableGuardImpl) NewStateChan() chan struct{} { g.mu.Lock() defer g.mu.Unlock() @@ -431,9 +451,11 @@ func (g *lockTableGuardImpl) isSameTxnAsReservation(ws waitingState) bool { // Finds the next lock, after the current one, to actively wait at. If it // finds the next lock the request starts actively waiting there, else it is -// told that it is done waiting. +// told that it is done waiting. lockTableImpl.finalizedTxnCache is used to +// accumulate intents to resolve. // Acquires g.mu. func (g *lockTableGuardImpl) findNextLockAfter(notify bool) { + g.toResolve = g.toResolve[:0] spans := g.spans.GetSpans(g.sa, g.ss) var span *spanset.Span resumingInSameSpan := false @@ -475,10 +497,24 @@ func (g *lockTableGuardImpl) findNextLockAfter(notify bool) { resumingInSameSpan = false span = stepToNextSpan(g) } + if len(g.toResolve) > 0 { + for i := range g.toResolve { + // TODO + if err := g.lt.UpdateLocks(&g.toResolve[i]); err != nil { + panic(err.Error()) + } + } + } g.mu.Lock() defer g.mu.Unlock() g.mu.state = waitingState{kind: doneWaiting} if notify { + if len(g.toResolve) > 0 { + // Force caller to release latches and resolve intents. The first + // state it will see after releasing latches is doneWaiting, which + // will cause it to resolve intents. + g.mu.startWait = true + } g.notify() } } @@ -1028,8 +1064,57 @@ func (l *lockState) clearLockHolder() { // it is set to false when the call to tryActiveWait is happening due to an // event for a different request or transaction (like a lock release) since in // that case the channel is notified first and the call to tryActiveWait() -// happens later in lockTableGuard.CurState(). The return value is true iff -// it is actively waiting. +// happens later in lockTableGuard.CurState(). +// +// It uses the finalizedTxnCache to decide that the caller does not need to +// wait on a lock of a transaction that is already finalized. +// +// - For unreplicated locks, this method will silently remove the lock and +// proceed as normal. +// - For replicated locks the behavior is more complicated since we need to +// resolve the intent. We desire: +// A. batching of intent resolution. +// B. minimize races where intent resolution is being performed by multiple +// requests. +// C. minimize races where intent has not yet been resolved but has been +// removed from the lock table, thereby causing some other request to +// evaluate wastefully and discover the intent. +// +// For A, the caller of tryActiveWait will accumulate the LockUpdates. For B, +// we only generate a LockUpdate here if this request is either a reader, or +// the first writer in the queue, i.e., it is only blocked by the lock +// holder. This prevents races between multiple writers in doing resolution +// but not between multiple readers and between readers and writers. We could +// be more conservative in only doing the intent resolution if the waiter was +// equivalent to a distinguished-waiter, but there it no guarantee that that +// distinguished waiter will do intent resolution in a timely manner (since +// it could block waiting on some other lock). Instead, the caller of +// tryActiveWait makes a best-effort to reduce racing (explained below). For +// C, the caller of tryActiveWait removes the lock from the in-memory +// data-structure only if the request does not need to wait anywhere, which +// means it will immediately proceed to intent resolution. Additionally, if +// the lock has already been removed, it suggests that some other request has +// already claimed intent resolution (or done it), so this request does not +// need to do the resolution. +// +// Ideally, we would strengthen B and C -- a request should make a claim on +// intent resolution for a set of keys, and will either resolve the intent, +// or due to an error will return that claim so others can do so. A +// replicated lock (intent) would not be removed from the in-memory +// data-structure until it was actually gone. +// TODO(sumeer): do this cleaner solution for batched intent resolution. +// +// In the future we'd like to augment the lockTable with an understanding of +// finalized but not yet resolved locks. These locks will allow conflicting +// transactions to proceed with evaluation without the need to first remove +// all traces of them via a round of replication. This is discussed in more +// detail in #41720. Specifically, see mention of "contention footprint" and +// COMMITTED_BUT_NOT_REMOVABLE. +// Also, resolving these locks/intents would proceed without latching, so we +// would not rely on MVCC scanning to add discovered locks to the lock table, +// since the discovered locks may be stale. +// +// The return value is true iff it is actively waiting. // Acquires l.mu, g.mu. func (l *lockState) tryActiveWait(g *lockTableGuardImpl, sa spanset.SpanAccess, notify bool) bool { l.mu.Lock() @@ -1047,6 +1132,23 @@ func (l *lockState) tryActiveWait(g *lockTableGuardImpl, sa spanset.SpanAccess, return false } + var replicatedLockFinalizedTxn *roachpb.Transaction + if lockHolderTxn != nil { + finalizedTxn, ok := g.lt.finalizedTxnCache.get(lockHolderTxn.ID) + if ok { + if l.holder.holder[lock.Replicated].txn == nil { + // Only held unreplicated. Release immediately. We don't expect the + // caller to GC this lockState and instead will GC it in + // tryUpdateLock. Note that there may be other waiters, so the caller + // may have to wait behind them. + l.clearLockHolder() + l.lockIsFree() + } else { + replicatedLockFinalizedTxn = finalizedTxn + } + } + } + if sa == spanset.SpanReadOnly { if lockHolderTxn == nil { // Reads only care about locker, not a reservation. @@ -1112,8 +1214,8 @@ func (l *lockState) tryActiveWait(g *lockTableGuardImpl, sa spanset.SpanAccess, return false } - // Need to wait. - + // May need to wait. + wait := true g.mu.Lock() defer g.mu.Unlock() if sa == spanset.SpanReadWrite { @@ -1131,7 +1233,13 @@ func (l *lockState) tryActiveWait(g *lockTableGuardImpl, sa spanset.SpanAccess, if qg == nil { panic("lockTable bug") } - qg.active = true + active := true + if replicatedLockFinalizedTxn != nil && l.queuedWriters.Front().Value.(*queuedGuard) == qg { + // First waiter, so should not wait. + active = false + wait = false + } + qg.active = active } else { // Not in queue so insert as active waiter. qg := &queuedGuard{ @@ -1139,6 +1247,11 @@ func (l *lockState) tryActiveWait(g *lockTableGuardImpl, sa spanset.SpanAccess, active: true, } if l.queuedWriters.Len() == 0 { + if replicatedLockFinalizedTxn != nil { + // First waiter, so should not wait. + qg.active = false + wait = false + } l.queuedWriters.PushFront(qg) } else { var e *list.Element @@ -1149,6 +1262,11 @@ func (l *lockState) tryActiveWait(g *lockTableGuardImpl, sa spanset.SpanAccess, } } if e == nil { + if replicatedLockFinalizedTxn != nil { + // First waiter, so should not wait. + qg.active = false + wait = false + } l.queuedWriters.PushFront(qg) } else { l.queuedWriters.InsertAfter(qg, e) @@ -1157,8 +1275,19 @@ func (l *lockState) tryActiveWait(g *lockTableGuardImpl, sa spanset.SpanAccess, g.mu.locks[l] = struct{}{} } } else { - l.waitingReaders.PushFront(g) - g.mu.locks[l] = struct{}{} + if replicatedLockFinalizedTxn != nil { + // Don't add to waitingReaders since all readers in waitingReaders are + // active waiters, and this request is not an active waiter here. + wait = false + } else { + l.waitingReaders.PushFront(g) + g.mu.locks[l] = struct{}{} + } + } + if !wait { + g.toResolve = append( + g.toResolve, roachpb.MakeLockUpdate(replicatedLockFinalizedTxn, roachpb.Span{Key: l.key})) + return false } // Make it an active waiter. g.key = l.key @@ -1779,6 +1908,7 @@ func (t *lockTableImpl) ScanAndEnqueue(req Request, guard lockTableGuard) lockTa if guard == nil { g = newLockTableGuardImpl() g.seqNum = atomic.AddUint64(&t.seqNum, 1) + g.lt = t g.txn = req.txnMeta() g.spans = req.LockSpans g.readTS = req.readConflictTimestamp() @@ -2102,6 +2232,11 @@ func stepToNextSpan(g *lockTableGuardImpl) *spanset.Span { return nil } +// TransactionIsFinalized implements the lockTable interface. +func (t *lockTableImpl) TransactionIsFinalized(txn *roachpb.Transaction) { + t.finalizedTxnCache.add(txn) +} + // Enable implements the lockTable interface. func (t *lockTableImpl) Enable(seq roachpb.LeaseSequence) { // Avoid disrupting other requests if the lockTable is already enabled. @@ -2129,6 +2264,9 @@ func (t *lockTableImpl) Clear(disable bool) { t.enabled = false } t.tryClearLocks(true /* force */) + // Also clear the finalized txn cache, since it won't be needed any time + // soon and consumes memory. + t.finalizedTxnCache.clear() } // For tests. diff --git a/pkg/kv/kvserver/concurrency/lock_table_waiter.go b/pkg/kv/kvserver/concurrency/lock_table_waiter.go index c0758b87bfb7..7bec15e641a3 100644 --- a/pkg/kv/kvserver/concurrency/lock_table_waiter.go +++ b/pkg/kv/kvserver/concurrency/lock_table_waiter.go @@ -95,17 +95,7 @@ type lockTableWaiterImpl struct { stopper *stop.Stopper ir IntentResolver lm LockManager - - // finalizedTxnCache is a small LRU cache that tracks transactions that - // were pushed and found to be finalized (COMMITTED or ABORTED). It is - // used as an optimization to avoid repeatedly pushing the transaction - // record when cleaning up the intents of an abandoned transaction. - // - // NOTE: it probably makes sense to maintain a single finalizedTxnCache - // across all Ranges on a Store instead of an individual cache per - // Range. For now, we don't do this because we don't share any state - // between separate concurrency.Manager instances. - finalizedTxnCache txnCache + lt lockTable // When set, WriteIntentError are propagated instead of pushing // conflicting transactions. @@ -142,16 +132,6 @@ func (w *lockTableWaiterImpl) WaitOn( var timer *timeutil.Timer var timerC <-chan time.Time var timerWaitingState waitingState - // Used to defer the resolution of duplicate intents. Intended to allow - // batching of intent resolution while cleaning up after abandoned txns. A - // request may begin deferring intent resolution and then be forced to wait - // again on other locks. This is ok, as the request that deferred intent - // resolution will often be the new reservation holder for those intents' - // keys. Even when this is not the case (e.g. the request is read-only so it - // can't hold reservations), any other requests that slip ahead will simply - // re-discover the intent(s) during evaluation and resolve them themselves. - var deferredResolution []roachpb.LockUpdate - defer w.resolveDeferredIntents(ctx, &err, &deferredResolution) for { select { case <-newStateC: @@ -219,54 +199,6 @@ func (w *lockTableWaiterImpl) WaitOn( continue } - // If we know that a lock holder is already finalized (COMMITTED - // or ABORTED), there's no reason to push it again. Instead, we - // can skip directly to intent resolution. - // - // As an optimization, we defer the intent resolution until the - // we're done waiting on all conflicting locks in this function. - // This allows us to accumulate a group of intents to resolve - // and send them together as a batch. - // - // Remember that if the lock is held, there will be at least one - // waiter with livenessPush = true (the distinguished waiter), - // so at least one request will enter this branch and perform - // the cleanup on behalf of all other waiters. - if livenessPush { - if pusheeTxn, ok := w.finalizedTxnCache.get(state.txn.ID); ok { - resolve := roachpb.MakeLockUpdate(pusheeTxn, roachpb.Span{Key: state.key}) - deferredResolution = append(deferredResolution, resolve) - - // Inform the LockManager that the lock has been updated with a - // finalized status so that it gets removed from the lockTable - // and we are allowed to proceed. - // - // For unreplicated locks, this is all that is needed - the - // lockTable is the source of truth so, once removed, the - // unreplicated lock is gone. It is perfectly valid for us to - // instruct the lock to be released because we know that the - // lock's owner is finalized. - // - // For replicated locks, this is a bit of a lie. The lock hasn't - // actually been updated yet, but we will be conducting intent - // resolution in the future (before we observe the corresponding - // MVCC state). This is safe because we already handle cases - // where locks exist only in the MVCC keyspace and not in the - // lockTable. - // - // In the future, we'd like to make this more explicit. - // Specifically, we'd like to augment the lockTable with an - // understanding of finalized but not yet resolved locks. These - // locks will allow conflicting transactions to proceed with - // evaluation without the need to first remove all traces of - // them via a round of replication. This is discussed in more - // detail in #41720. Specifically, see mention of "contention - // footprint" and COMMITTED_BUT_NOT_REMOVABLE. - w.lm.OnLockUpdated(ctx, &deferredResolution[len(deferredResolution)-1]) - continue - } - } - // The request should push to detect abandoned locks due to // failed transaction coordinators, detect deadlocks between // transactions, or both, but only after delay. This delay @@ -334,6 +266,8 @@ func (w *lockTableWaiterImpl) WaitOn( // waiting, re-acquire latches, and check the lockTable again for // any new conflicts. If it find none, it can proceed with // evaluation. + toResolve := guard.ResolveBeforeScanning() + w.resolveDeferredIntents(ctx, &err, &toResolve) return nil default: @@ -415,11 +349,6 @@ func (w *lockTableWaiterImpl) WaitOnLock( }) } -// ClearCaches implements the lockTableWaiter interface. -func (w *lockTableWaiterImpl) ClearCaches() { - w.finalizedTxnCache.clear() -} - // pushLockTxn pushes the holder of the provided lock. // // The method blocks until the lock holder transaction experiences a state @@ -482,7 +411,7 @@ func (w *lockTableWaiterImpl) pushLockTxn( // avoids needing to push it again if we find another one of its locks and // allows for batching of intent resolution. if pusheeTxn.Status.IsFinalized() { - w.finalizedTxnCache.add(pusheeTxn) + w.lt.TransactionIsFinalized(pusheeTxn) } // If the push succeeded then the lock holder transaction must have diff --git a/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go b/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go index 0f1f44deb0fc..9db04855a8c4 100644 --- a/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go +++ b/pkg/kv/kvserver/concurrency/lock_table_waiter_test.go @@ -72,6 +72,9 @@ func (g *mockLockTableGuard) CurState() waitingState { } return s } +func (g *mockLockTableGuard) ResolveBeforeScanning() []roachpb.LockUpdate { + return nil +} func (g *mockLockTableGuard) notify() { g.signal <- struct{}{} } // mockLockTableGuard implements the LockManager interface. @@ -93,11 +96,13 @@ func setupLockTableWaiterTest() (*lockTableWaiterImpl, *mockIntentResolver, *moc guard := &mockLockTableGuard{ signal: make(chan struct{}, 1), } + // TODO: needs a lt. w := &lockTableWaiterImpl{ st: st, stopper: stop.NewStopper(), ir: ir, lm: guard, + lt: &lockTableImpl{}, } return w, ir, guard } @@ -548,7 +553,8 @@ func TestLockTableWaiterDeferredIntentResolverError(t *testing.T) { // Add the conflicting txn to the finalizedTxnCache so that the request // avoids the transaction record push and defers the intent resolution. pusheeTxn.Status = roachpb.ABORTED - w.finalizedTxnCache.add(&pusheeTxn) + // TODO: fix + // w.finalizedTxnCache.add(&pusheeTxn) g.state = waitingState{ kind: waitForDistinguished, diff --git a/pkg/kv/kvserver/concurrency/testdata/concurrency_manager/clear_abandoned_intents b/pkg/kv/kvserver/concurrency/testdata/concurrency_manager/clear_abandoned_intents index 4631542a93f0..e8f385c34e73 100644 --- a/pkg/kv/kvserver/concurrency/testdata/concurrency_manager/clear_abandoned_intents +++ b/pkg/kv/kvserver/concurrency/testdata/concurrency_manager/clear_abandoned_intents @@ -257,3 +257,421 @@ finish req=req1 reset namespace ---- + +# TODO: the current output is based on the code in master. + +# Does resolution for unreplicated locks, even though could just simply remove +# them from lock table. + +new-txn name=txn1 ts=10,1 epoch=0 +---- + +new-txn name=txn2 ts=10,1 epoch=0 +---- + +new-request name=req1 txn=txn1 ts=10,1 + scan key=a endkey=z +---- + +sequence req=req1 +---- +[1] sequence req1: sequencing request +[1] sequence req1: acquiring latches +[1] sequence req1: scanning lock table for conflicting locks +[1] sequence req1: sequencing complete, returned guard + +handle-write-intent-error req=req1 lease-seq=1 + intent txn=txn2 key=a + intent txn=txn2 key=b +---- +[2] handle write intent error req1: handled conflicting intents on "a", "b", released latches + +debug-lock-table +---- +global: num=2 + lock: "a" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + lock: "b" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] +local: num=0 + +new-request name=req2 txn=txn2 ts=10,1 + put key=g value=v1 + put key=h value=v2 +---- + +sequence req=req2 +---- +[3] sequence req2: sequencing request +[3] sequence req2: acquiring latches +[3] sequence req2: scanning lock table for conflicting locks +[3] sequence req2: sequencing complete, returned guard + +on-lock-acquired req=req2 key=g dur=u +---- +[-] acquire lock: txn 00000002 @ g + +on-lock-acquired req=req2 key=h dur=u +---- +[-] acquire lock: txn 00000002 @ h + +finish req=req2 +---- +[-] finish req2: finishing request + +debug-lock-table +---- +global: num=4 + lock: "a" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + lock: "b" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + lock: "g" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] + lock: "h" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] +local: num=0 + +sequence req=req1 +---- +[4] sequence req1: re-sequencing request +[4] sequence req1: acquiring latches +[4] sequence req1: scanning lock table for conflicting locks +[4] sequence req1: waiting in lock wait-queues +[4] sequence req1: pushing timestamp of txn 00000002 above 0.000000010,1 +[4] sequence req1: blocked on select in concurrency_test.(*cluster).PushTransaction + +debug-lock-table +---- +global: num=4 + lock: "a" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + waiting readers: + req: 3, txn: 00000001-0000-0000-0000-000000000000 + distinguished req: 3 + lock: "b" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + lock: "g" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] + lock: "h" + holder: txn: 00000002-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] +local: num=0 + +on-txn-updated txn=txn2 status=aborted +---- +[-] update txn: aborting txn2 +[4] sequence req1: resolving intent "a" for txn 00000002 with ABORTED status +[4] sequence req1: resolving a batch of 3 intent(s) +[4] sequence req1: resolving intent "b" for txn 00000002 with ABORTED status +[4] sequence req1: resolving intent "g" for txn 00000002 with ABORTED status +[4] sequence req1: resolving intent "h" for txn 00000002 with ABORTED status +[4] sequence req1: acquiring latches +[4] sequence req1: scanning lock table for conflicting locks +[4] sequence req1: sequencing complete, returned guard + +debug-lock-table +---- +global: num=0 +local: num=0 + +finish req=req1 +---- +[-] finish req1: finishing request + +reset namespace +---- + +# TODO: the current output is based on the code in master. + +# txn1's req clears an intent from the lock table without resolving it, causing +# txn2's req to discover it again. + +new-txn name=txn1 ts=12,1 epoch=0 +---- + +new-txn name=txn2 ts=11,1 epoch=0 +---- + +new-txn name=txn3 ts=10,1 epoch=0 +---- + +new-txn name=txn4 ts=10,1 epoch=0 +---- + +new-txn name=txn5 ts=10,1 epoch=0 +---- + +new-request name=req1 txn=txn1 ts=12,1 + put key=c value=v1 + put key=d value=v1 + put key=e value=v1 +---- + +sequence req=req1 +---- +[1] sequence req1: sequencing request +[1] sequence req1: acquiring latches +[1] sequence req1: scanning lock table for conflicting locks +[1] sequence req1: sequencing complete, returned guard + +# Normally req1 will not discover write intents for c, d, e in one shot, but +# we do this for shortening the test. +handle-write-intent-error req=req1 lease-seq=1 + intent txn=txn3 key=c + intent txn=txn3 key=d + intent txn=txn5 key=e +---- +[2] handle write intent error req1: handled conflicting intents on "c", "d", "e", released latches + +debug-lock-table +---- +global: num=3 + lock: "c" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 + lock: "d" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 + lock: "e" + holder: txn: 00000005-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 +local: num=0 + +sequence req=req1 +---- +[3] sequence req1: re-sequencing request +[3] sequence req1: acquiring latches +[3] sequence req1: scanning lock table for conflicting locks +[3] sequence req1: waiting in lock wait-queues +[3] sequence req1: pushing txn 00000003 to abort +[3] sequence req1: blocked on select in concurrency_test.(*cluster).PushTransaction + +debug-lock-table +---- +global: num=3 + lock: "c" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: true req: 5, txn: 00000001-0000-0000-0000-000000000000 + distinguished req: 5 + lock: "d" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 + lock: "e" + holder: txn: 00000005-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 +local: num=0 + +new-request name=req3 txn=txn3 ts=10,1 + put key=a value=v3 +---- + +sequence req=req3 +---- +[4] sequence req3: sequencing request +[4] sequence req3: acquiring latches +[4] sequence req3: scanning lock table for conflicting locks +[4] sequence req3: sequencing complete, returned guard + +on-lock-acquired req=req3 key=a dur=u +---- +[-] acquire lock: txn 00000003 @ a + +finish req=req3 +---- +[-] finish req3: finishing request + +new-request name=req4 txn=txn4 ts=10,1 + put key=b value=v4 +---- + +sequence req=req4 +---- +[5] sequence req4: sequencing request +[5] sequence req4: acquiring latches +[5] sequence req4: scanning lock table for conflicting locks +[5] sequence req4: sequencing complete, returned guard + +on-lock-acquired req=req4 key=b dur=u +---- +[-] acquire lock: txn 00000004 @ b + +finish req=req4 +---- +[-] finish req4: finishing request + +debug-lock-table +---- +global: num=5 + lock: "a" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] + lock: "b" + holder: txn: 00000004-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] + lock: "c" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: true req: 5, txn: 00000001-0000-0000-0000-000000000000 + distinguished req: 5 + lock: "d" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 + lock: "e" + holder: txn: 00000005-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 +local: num=0 + +new-request name=req2 txn=txn2 ts=11,1 + scan key=a endkey=c + scan key=d endkey=e +---- + +sequence req=req2 +---- +[6] sequence req2: sequencing request +[6] sequence req2: acquiring latches +[6] sequence req2: scanning lock table for conflicting locks +[6] sequence req2: waiting in lock wait-queues +[6] sequence req2: pushing timestamp of txn 00000003 above 0.000000011,1 +[6] sequence req2: blocked on select in concurrency_test.(*cluster).PushTransaction + +debug-lock-table +---- +global: num=5 + lock: "a" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] + waiting readers: + req: 8, txn: 00000002-0000-0000-0000-000000000000 + distinguished req: 8 + lock: "b" + holder: txn: 00000004-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] + lock: "c" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: true req: 5, txn: 00000001-0000-0000-0000-000000000000 + distinguished req: 5 + lock: "d" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 + lock: "e" + holder: txn: 00000005-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 +local: num=0 + +# req1 resolves intent c and removes intent d from the lock table, and waits at e. +on-txn-updated txn=txn3 status=aborted +---- +[-] update txn: aborting txn3 +[3] sequence req1: resolving intent "c" for txn 00000003 with ABORTED status +[3] sequence req1: pushing txn 00000005 to abort +[3] sequence req1: blocked on select in concurrency_test.(*cluster).PushTransaction +[6] sequence req2: resolving intent "a" for txn 00000003 with ABORTED status +[6] sequence req2: pushing timestamp of txn 00000004 above 0.000000011,1 +[6] sequence req2: blocked on select in concurrency_test.(*cluster).PushTransaction + +debug-lock-table +---- +global: num=4 + lock: "b" + holder: txn: 00000004-0000-0000-0000-000000000000, ts: 0.000000010,1, info: unrepl epoch: 0, seqs: [0] + waiting readers: + req: 8, txn: 00000002-0000-0000-0000-000000000000 + distinguished req: 8 + lock: "c" + res: req: 5, txn: 00000001-0000-0000-0000-000000000000, ts: 0.000000012,1, seq: 0 + lock: "d" + res: req: 5, txn: 00000001-0000-0000-0000-000000000000, ts: 0.000000012,1, seq: 0 + lock: "e" + holder: txn: 00000005-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: true req: 5, txn: 00000001-0000-0000-0000-000000000000 + distinguished req: 5 +local: num=0 + +# req2 thinks it does not need to wait because b is resolved, and d has been removed +# from in-memory lock table state. +on-txn-updated txn=txn4 status=aborted +---- +[-] update txn: aborting txn4 +[6] sequence req2: resolving intent "b" for txn 00000004 with ABORTED status +[6] sequence req2: acquiring latches +[6] sequence req2: scanning lock table for conflicting locks +[6] sequence req2: sequencing complete, returned guard + +debug-lock-table +---- +global: num=3 + lock: "c" + res: req: 5, txn: 00000001-0000-0000-0000-000000000000, ts: 0.000000012,1, seq: 0 + lock: "d" + res: req: 5, txn: 00000001-0000-0000-0000-000000000000, ts: 0.000000012,1, seq: 0 + lock: "e" + holder: txn: 00000005-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: true req: 5, txn: 00000001-0000-0000-0000-000000000000 + distinguished req: 5 +local: num=0 + +# req2 discovers lock at d. +handle-write-intent-error req=req2 lease-seq=1 + intent txn=txn3 key=d +---- +[7] handle write intent error req2: handled conflicting intents on "d", released latches + +debug-lock-table +---- +global: num=3 + lock: "c" + res: req: 5, txn: 00000001-0000-0000-0000-000000000000, ts: 0.000000012,1, seq: 0 + lock: "d" + holder: txn: 00000003-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: false req: 5, txn: 00000001-0000-0000-0000-000000000000 + lock: "e" + holder: txn: 00000005-0000-0000-0000-000000000000, ts: 0.000000010,1, info: repl epoch: 0, seqs: [0] + queued writers: + active: true req: 5, txn: 00000001-0000-0000-0000-000000000000 + distinguished req: 5 +local: num=0 + +# req2 now resolves lock at d. +sequence req=req2 +---- +[8] sequence req2: re-sequencing request +[8] sequence req2: acquiring latches +[8] sequence req2: scanning lock table for conflicting locks +[8] sequence req2: waiting in lock wait-queues +[8] sequence req2: resolving a batch of 1 intent(s) +[8] sequence req2: resolving intent "d" for txn 00000003 with ABORTED status +[8] sequence req2: acquiring latches +[8] sequence req2: scanning lock table for conflicting locks +[8] sequence req2: sequencing complete, returned guard + +finish req=req2 +---- +[-] finish req2: finishing request + +on-txn-updated txn=txn5 status=aborted +---- +[-] update txn: aborting txn5 +[3] sequence req1: resolving intent "e" for txn 00000005 with ABORTED status +[3] sequence req1: resolving a batch of 1 intent(s) +[3] sequence req1: resolving intent "d" for txn 00000003 with ABORTED status +[3] sequence req1: acquiring latches +[3] sequence req1: scanning lock table for conflicting locks +[3] sequence req1: sequencing complete, returned guard + +finish req=req1 +---- +[-] finish req1: finishing request + +reset namespace +----