Skip to content

Commit

Permalink
roachpb: add Leader lease type definition
Browse files Browse the repository at this point in the history 
Fixes cockroachdb#125225.

This commit adds a new Term field to the Lease struct. This field
defines the term of the raft leader that a leader lease is associated
with. The lease is valid for as long as the raft leader has a guarantee
from store liveness that it remains the leader under this term. The
lease is invalid if the raft leader loses leadership (i.e. changes its
term).

The field is not yet used.

Release note: None
  • Loading branch information
@nvanbenschoten
nvanbenschoten committed Jul 2, 2024
1 parent 757ebff commit 41c2a01
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 21 deletions.
79 changes: 65 additions & 14 deletions pkg/roachpb/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1853,10 +1853,15 @@ func (l Lease) SafeFormat(w redact.SafePrinter, _ rune) {
return
}
w.Printf("repl=%s seq=%d start=%s", l.Replica, l.Sequence, l.Start)
if l.Type() == LeaseExpiration {
switch l.Type() {
case LeaseExpiration:
w.Printf(" exp=%s", l.Expiration)
} else {
case LeaseEpoch:
w.Printf(" epo=%d min-exp=%s", l.Epoch, l.MinExpiration)
case LeaseLeader:
w.Printf(" term=%d min-exp=%s", l.Term, l.MinExpiration)
default:
panic("unexpected lease type")
}
w.Printf(" pro=%s", l.ProposedTS)
}
Expand All @@ -1883,14 +1888,23 @@ const (
// LeaseEpoch allows range operations while the node liveness epoch
// is equal to the lease epoch.
LeaseEpoch
// LeaseLeader allows range operations while the replica is guaranteed
// to be the range's raft leader.
LeaseLeader
)

// Type returns the lease type.
func (l Lease) Type() LeaseType {
if l.Epoch == 0 {
return LeaseExpiration
if l.Epoch != 0 && l.Term != 0 {
panic("lease cannot have both epoch and term")
}
if l.Epoch != 0 {
return LeaseEpoch
}
return LeaseEpoch
if l.Term != 0 {
return LeaseLeader
}
return LeaseExpiration
}

// Speculative returns true if this lease instance doesn't correspond to a
Expand All @@ -1914,7 +1928,9 @@ func (l Lease) Speculative() bool {
// expToEpochEquiv indicates whether an expiration-based lease
// can be considered equivalent to an epoch-based lease during
// a promotion from expiration-based to epoch-based. It is used
// for mixed-version compatibility.
// for mixed-version compatibility. No such flag is needed for
// expiration-based to leader lease promotion, because there is
// no need for mixed-version compatibility.
//
// NB: Lease.Equivalent is NOT symmetric. For expiration-based
// leases, a lease is equivalent to another with an equal or
Expand All @@ -1935,14 +1951,14 @@ func (l Lease) Speculative() bool {
// times are the same, the leases could turn out to be non-equivalent -- in
// that case they will share a start time but not the sequence.
//
// NB: we do not allow transitions from epoch-based or leader leases (not
// yet implemented) to expiration-based leases to be equivalent. This was
// because both of the former lease types don't have an expiration in the
// lease, while the latter does. We can introduce safety violations by
// shortening the lease expiration if we allow this transition, since the
// new lease may not apply at the leaseholder until much after it applies at
// some other replica, so the leaseholder may continue acting as one based
// on an old lease, while the other replica has stepped up as leaseholder.
// NB: we do not allow transitions from epoch-based or leader leases to
// expiration-based leases to be equivalent. This was because both of the
// former lease types don't have an expiration in the lease, while the
// latter does. We can introduce safety violations by shortening the lease
// expiration if we allow this transition, since the new lease may not apply
// at the leaseholder until much after it applies at some other replica, so
// the leaseholder may continue acting as one based on an old lease, while
// the other replica has stepped up as leaseholder.
func (l Lease) Equivalent(newL Lease, expToEpochEquiv bool) bool {
// Ignore proposed timestamp & deprecated start stasis.
l.ProposedTS, newL.ProposedTS = hlc.ClockTimestamp{}, hlc.ClockTimestamp{}
Expand Down Expand Up @@ -1977,6 +1993,17 @@ func (l Lease) Equivalent(newL Lease, expToEpochEquiv bool) bool {
if l.MinExpiration.LessEq(newL.MinExpiration) {
l.MinExpiration, newL.MinExpiration = hlc.Timestamp{}, hlc.Timestamp{}
}

case LeaseLeader:
if l.Term == newL.Term {
l.Term, newL.Term = 0, 0
}
// For leader leases, extensions to the minimum expiration are considered
// equivalent.
if l.MinExpiration.LessEq(newL.MinExpiration) {
l.MinExpiration, newL.MinExpiration = hlc.Timestamp{}, hlc.Timestamp{}
}

case LeaseExpiration:
switch newL.Type() {
case LeaseEpoch:
Expand All @@ -1999,6 +2026,27 @@ func (l Lease) Equivalent(newL Lease, expToEpochEquiv bool) bool {
newL.MinExpiration = hlc.Timestamp{}
}

case LeaseLeader:
// An expiration-based lease being promoted to a leader lease. This
// transition occurs after a successful lease transfer if the setting
// kv.transfer_expiration_leases_first.enabled is enabled and leader
// leases are in use.
//
// Expiration-based leases carry a local expiration timestamp. Leader
// leases extend their expiration indirectly through the leadership
// fortification protocol and associated Store Liveness heartbeats. We
// assume that this promotion is only proposed if the leader support
// expiration (and associated min expiration) is equal to or later than
// previous expiration carried by the expiration-based lease. This is a
// case where Equivalent is not commutative, as the reverse transition
// (from leader lease to expiration-based) requires a sequence increment.
//
// Ignore expiration, term, and min expiration. The remaining fields
// which are compared are Replica and Start.
l.Expiration = nil
newL.Term = 0
newL.MinExpiration = hlc.Timestamp{}

case LeaseExpiration:
// See the comment above, though this field's nullability wasn't
// changed. We nil it out for completeness only.
Expand Down Expand Up @@ -2090,6 +2138,9 @@ func (l *Lease) Equal(that interface{}) bool {
if !l.MinExpiration.Equal(&that1.MinExpiration) {
return false
}
if l.Term != that1.Term {
return false
}
return true
}

Expand Down
25 changes: 19 additions & 6 deletions pkg/roachpb/data.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -704,9 +704,13 @@ message Lease {
(gogoproto.nullable) = false,
(gogoproto.casttype) = "github.com/cockroachdb/cockroach/pkg/util/hlc.ClockTimestamp"];

// The epoch of the lease holder's node liveness entry. This field is only set
// for epoch-based leases. If this value is non-zero, the expiration field and
// the term field (not yet implemented) are ignored.
// The epoch of the lease holder's node liveness record. The lease inherits
// the expiration of the node liveness record for as long as the node liveness
// record retains this epoch. The lease is invalid if the node liveness record
// is updated with a different epoch.
//
// This field is only set for epoch-based leases. If this value is non-zero,
// the expiration field and the term field must not be set.
int64 epoch = 6;

// A zero-indexed sequence number which is incremented during the acquisition
Expand All @@ -726,13 +730,22 @@ message Lease {

// The minimum expiration at which the lease expires, independent of any other
// expiry condition. This field can be used to place a floor on the expiration
// for epoch-based leases and leader leases (not yet implemented) to prevent
// expiration regressions when upgrading from an expiration-based lease. It is
// not supported for expiration-based leases.
// for epoch-based leases and leader leases to prevent expiration regressions
// when upgrading from an expiration-based lease. It is not supported for
// expiration-based leases.
//
// Like expiration above, this is an exclusive value, i.e. the lease is valid
// in [start, max(min_expiration, <expiration from epoch or term>)).
util.hlc.Timestamp min_expiration = 9 [(gogoproto.nullable) = false];

// The term of the raft leader that a leader lease is associated with. The
// lease is valid for as long as the raft leader has a guarantee from store
// liveness that it remains the leader under this term. The lease is invalid
// if the raft leader loses leadership (i.e. changes its term).
//
// This field is only set for leader leases. If non-zero, the expiration field
// and the epoch field must not be set.
uint64 term = 10;
}

// AbortSpanEntry contains information about a transaction which has
Expand Down
54 changes: 53 additions & 1 deletion pkg/roachpb/data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1121,6 +1121,22 @@ func TestLeaseStringAndSafeFormat(t *testing.T) {
},
exp: "repl=(n1,s1):1 seq=3 start=0.000000001,1 epo=4 min-exp=0.000000002,1 pro=0.000000001,0",
},
{
name: "leader",
lease: Lease{
Replica: ReplicaDescriptor{
NodeID: 1,
StoreID: 1,
ReplicaID: 1,
},
Start: makeClockTS(1, 1),
ProposedTS: makeClockTS(1, 0),
Sequence: 3,
MinExpiration: makeTS(2, 1),
Term: 5,
},
exp: "repl=(n1,s1):1 seq=3 start=0.000000001,1 term=5 min-exp=0.000000002,1 pro=0.000000001,0",
},
} {
t.Run(tc.name, func(t *testing.T) {
// String.
Expand All @@ -1133,6 +1149,13 @@ func TestLeaseStringAndSafeFormat(t *testing.T) {
}
}

func TestLeaseType(t *testing.T) {
require.Equal(t, LeaseExpiration, Lease{}.Type())
require.Equal(t, LeaseEpoch, Lease{Epoch: 1}.Type())
require.Equal(t, LeaseLeader, Lease{Term: 1}.Type())
require.Panics(t, func() { Lease{Epoch: 1, Term: 1}.Type() })
}

func TestLeaseEquivalence(t *testing.T) {
r1 := ReplicaDescriptor{NodeID: 1, StoreID: 1, ReplicaID: 1}
r2 := ReplicaDescriptor{NodeID: 2, StoreID: 2, ReplicaID: 2}
Expand All @@ -1150,6 +1173,11 @@ func TestLeaseEquivalence(t *testing.T) {
expire1TS2 := Lease{Replica: r1, Start: ts2, Expiration: ts2.ToTimestamp().Clone()}
expire2 := Lease{Replica: r1, Start: ts1, Expiration: ts3.ToTimestamp().Clone()}
expire2R2TS2 := Lease{Replica: r2, Start: ts2, Expiration: ts3.ToTimestamp().Clone()}
leader1 := Lease{Replica: r1, Start: ts1, Term: 1}
leader1R2 := Lease{Replica: r2, Start: ts1, Term: 1}
leader1TS2 := Lease{Replica: r1, Start: ts2, Term: 1}
leader2 := Lease{Replica: r1, Start: ts1, Term: 2}
leader2R2TS2 := Lease{Replica: r2, Start: ts2, Term: 2}

proposed1 := Lease{Replica: r1, Start: ts1, Epoch: 1, ProposedTS: ts1}
proposed2 := Lease{Replica: r1, Start: ts1, Epoch: 2, ProposedTS: ts1}
Expand All @@ -1167,13 +1195,17 @@ func TestLeaseEquivalence(t *testing.T) {
epoch1MinExp2 := Lease{Replica: r1, Start: ts1, Epoch: 1, MinExpiration: ts2.ToTimestamp()}
epoch1MinExp3 := Lease{Replica: r1, Start: ts1, Epoch: 1, MinExpiration: ts3.ToTimestamp()}
epoch2MinExp2 := Lease{Replica: r1, Start: ts1, Epoch: 2, MinExpiration: ts2.ToTimestamp()}
leader1MinExp2 := Lease{Replica: r1, Start: ts1, Term: 1, MinExpiration: ts2.ToTimestamp()}
leader1MinExp3 := Lease{Replica: r1, Start: ts1, Term: 1, MinExpiration: ts3.ToTimestamp()}
leader2MinExp2 := Lease{Replica: r1, Start: ts1, Term: 2, MinExpiration: ts2.ToTimestamp()}

testCases := []struct {
l, ol Lease
expSuccess bool
}{
{epoch1, epoch1, true}, // same epoch lease
{expire1, expire1, true}, // same expiration lease
{leader1, leader1, true}, // same leader lease
{epoch1, epoch1R2, false}, // different epoch leases
{epoch1, epoch1TS2, false}, // different epoch leases
{epoch1, epoch2, false}, // different epoch leases
Expand All @@ -1183,20 +1215,28 @@ func TestLeaseEquivalence(t *testing.T) {
{expire1, expire2R2TS2, false}, // different expiration leases
{expire1, expire2, true}, // same expiration lease, extended
{expire2, expire1, false}, // same expiration lease, extended but backwards
{leader1, leader1R2, false}, // different leader leases
{leader1, leader1TS2, false}, // different leader leases
{leader1, leader2, false}, // different leader leases
{leader1, leader2R2TS2, false}, // different leader leases
{epoch1, expire1, false}, // epoch and expiration leases, same replica and start time
{epoch1, expire1R2, false}, // epoch and expiration leases, different replica
{epoch1, expire1TS2, false}, // epoch and expiration leases, different start time
{expire1, epoch1, true}, // expiration and epoch leases, same replica and start time
{expire1, epoch1R2, false}, // expiration and epoch leases, different replica
{expire1, epoch1TS2, false}, // expiration and epoch leases, different start time
{epoch1, leader1, false}, // epoch and leader leases, same replica and start time
{leader1, epoch1, false}, // leader and epoch leases, same replica and start time
{expire1, leader1, true}, // expiration and leader leases, same replica and start time
{leader1, expire1, false}, // leader and expiration leases, same replica and start time
{proposed1, proposed1, true}, // exact leases with identical timestamps
{proposed1, proposed2, false}, // same proposed timestamps, but diff epochs
{proposed1, proposed3, true}, // different proposed timestamps, same lease
{stasis1, stasis2, true}, // same lease, different stasis timestamps
{epoch1, epoch1Voter, true}, // same epoch lease, different replica type
{epoch1, epoch1Learner, true}, // same epoch lease, different replica type
{epoch1Voter, epoch1Learner, true}, // same epoch lease, different replica type
// Test minimum expiration.
// Test minimum expiration with epoch leases.
{epoch1, epoch1MinExp2, true}, // different epoch leases, newer min expiration
{epoch1, epoch1MinExp3, true}, // different epoch leases, newer min expiration
{epoch1MinExp2, epoch1, false}, // different epoch leases, older min expiration
Expand All @@ -1206,6 +1246,16 @@ func TestLeaseEquivalence(t *testing.T) {
{epoch1MinExp3, epoch1MinExp2, false}, // different epoch leases, older min expiration
{epoch1MinExp3, epoch1MinExp3, true}, // same epoch leases, same min expiration
{epoch1MinExp2, epoch2MinExp2, false}, // different epoch leases
// Test minimum expiration with leader leases.
{leader1, leader1MinExp2, true}, // different leader leases, newer min expiration
{leader1, leader1MinExp3, true}, // different leader leases, newer min expiration
{leader1MinExp2, leader1, false}, // different leader leases, older min expiration
{leader1MinExp2, leader1MinExp2, true}, // same leader leases, same min expiration
{leader1MinExp2, leader1MinExp3, true}, // different leader leases, newer min expiration
{leader1MinExp3, leader1, false}, // different leader leases, older min expiration
{leader1MinExp3, leader1MinExp2, false}, // different leader leases, older min expiration
{leader1MinExp3, leader1MinExp3, true}, // same leader leases, same min expiration
{leader1MinExp2, leader2MinExp2, false}, // different leader leases
}

for i, tc := range testCases {
Expand Down Expand Up @@ -1257,6 +1307,7 @@ func TestLeaseEqual(t *testing.T) {
Sequence LeaseSequence
AcquisitionType LeaseAcquisitionType
MinExpiration hlc.Timestamp
Term uint64
}
// Verify that the lease structure does not change unexpectedly. If a compile
// error occurs on the following line of code, update the expectedLease
Expand Down Expand Up @@ -1312,6 +1363,7 @@ func TestLeaseEqual(t *testing.T) {
{Sequence: 1},
{AcquisitionType: 1},
{MinExpiration: ts},
{Term: 1},
}
for _, c := range testCases {
t.Run("", func(t *testing.T) {
Expand Down

0 comments on commit 41c2a01

Please sign in to comment.