kvnemesis: validate writes and locking reads under weak isolation txns

Fixes cockroachdb#100169.

This commit updates kvnemesis to validate the atomicity of writes and locking
reads under weak isolation transactions, leaving only non-locking reads
unvalidated. This strengthens the validation of weak isolation transactions,
which were previously in the mix but ignored during validation. In the future,
we could explore adding some validation for non-locking reads as well, although
this will be both more complex and isolation level dependent (snapshot and read
committed behave differently).

This enhancement serves an important role in testing the correctness of
replicated locking reads under weak isolation transactions, both in that they
are never lost like unreplicated locks, and that they enforce isolation even
after being released by bumping the timestamp cache on release to the commit
timestamp (7702ff8 and 0e6de2b). If I remove either of those two commits,
kvnemesis now fails.

While here, the commit also bumps the maximum number of operations per
transaction from 3 to 5. This creates more interesting transactions.

Release note: None

pkg/kv/kvnemesis/generator.go

@@ -1371,7 +1371,8 @@ func makeClosureTxn(
 		var allowed []opGen
 		g.registerClientOps(&allowed, txnClientOps)
 		g.registerBatchOps(&allowed, txnBatchOps)
-		numOps := rng.Intn(4)
+		const maxOps = 5
+		numOps := rng.Intn(maxOps + 1)
 		ops := make([]Operation, numOps)
 		for i := range ops {
 			ops[i] = g.selectOp(rng, allowed)

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_atomic_locking_replicated_get

@@ -0,0 +1,13 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.GetForShareGuaranteedDurability(ctx, tk(1)) // (v2, <nil>)
+  txn.Put(ctx, tk(3), sv(3)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000003"/0.000000003,0 @ s3 v3

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_atomic_locking_replicated_get_and_missing_key

@@ -0,0 +1,13 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.GetForShareGuaranteedDurability(ctx, tk(1)) // (<nil>, <nil>)
+  txn.Put(ctx, tk(3), sv(3)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000003"/0.000000003,0 @ s3 v3

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_atomic_locking_replicated_scan

@@ -0,0 +1,15 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Put(ctx, tk(2), sv(3)) // @0.000000001,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.ScanForShareGuaranteedDurability(ctx, tk(1), tk(3), 0) // (/Table/100/"0000000000000001":v2, /Table/100/"0000000000000002":v3, <nil>)
+  txn.Put(ctx, tk(3), sv(4)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000002"/0.000000001,0 @ s3 v3
+/Table/100/"0000000000000003"/0.000000003,0 @ s4 v4

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_atomic_locking_replicated_scan_and_missing_key

@@ -0,0 +1,15 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Put(ctx, tk(2), sv(3)) // @0.000000001,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.ScanForShareGuaranteedDurability(ctx, tk(1), tk(3), 0) // (/Table/100/"0000000000000001":v2, <nil>)
+  txn.Put(ctx, tk(3), sv(4)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000002"/0.000000001,0 @ s3 v3
+/Table/100/"0000000000000003"/0.000000003,0 @ s4 v4

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_atomic_writes

@@ -0,0 +1,10 @@
+echo
+----
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.Put(ctx, tk(1), sv(1)) // <nil>
+  txn.Put(ctx, tk(2), sv(2)) // <nil>
+  return nil
+}) // @0.000000002,0 <nil>
+/Table/100/"0000000000000001"/0.000000002,0 @ s1 v1
+/Table/100/"0000000000000002"/0.000000002,0 @ s2 v2

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_deleterange

@@ -0,0 +1,15 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(2), sv(2)) // @0.000000002,0 <nil>
+db0.Put(ctx, tk(3), sv(3)) // @0.000000001,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.DelRange(ctx, tk(1), tk(4), true /* @s4 */) // (/Table/100/"0000000000000001", /Table/100/"0000000000000003", <nil>)
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000003,0 @ s4 v4
+/Table/100/"0000000000000002"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000003"/0.000000001,0 @ s3 v3
+/Table/100/"0000000000000003"/0.000000003,0 @ s4 v4

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_locking_replicated_get

@@ -0,0 +1,14 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.GetForShareGuaranteedDurability(ctx, tk(1)) // (v1, <nil>)
+  txn.Put(ctx, tk(3), sv(3)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000003"/0.000000003,0 @ s3 v3
+committed snapshot txn non-atomic timestamps: [r]/Table/100/"0000000000000001":[0.000000001,0, 0.000000002,0)->v1 [w]/Table/100/"0000000000000003":0.000000003,0->v3@s3

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_locking_replicated_scan

@@ -0,0 +1,16 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Put(ctx, tk(2), sv(3)) // @0.000000001,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.ScanForShareGuaranteedDurability(ctx, tk(1), tk(3), 0) // (/Table/100/"0000000000000001":v1, /Table/100/"0000000000000002":v3, <nil>)
+  txn.Put(ctx, tk(3), sv(4)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000002"/0.000000001,0 @ s3 v3
+/Table/100/"0000000000000003"/0.000000003,0 @ s4 v4
+committed snapshot txn non-atomic timestamps: [r]/Table/100/"0000000000000001":[0.000000001,0, 0.000000002,0)->v1 [r]/Table/100/"0000000000000002":[0.000000001,0, <max>)->v3 [w]/Table/100/"0000000000000003":0.000000003,0->v4@s4

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_locking_unreplicated_get

@@ -0,0 +1,13 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.GetForShare(ctx, tk(1)) // (v1, <nil>)
+  txn.Put(ctx, tk(3), sv(3)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000003"/0.000000003,0 @ s3 v3

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_locking_unreplicated_scan

@@ -0,0 +1,15 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Put(ctx, tk(2), sv(3)) // @0.000000001,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.ScanForShare(ctx, tk(1), tk(3), 0) // (/Table/100/"0000000000000001":v1, /Table/100/"0000000000000002":v3, <nil>)
+  txn.Put(ctx, tk(3), sv(4)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000002"/0.000000001,0 @ s3 v3
+/Table/100/"0000000000000003"/0.000000003,0 @ s4 v4

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_non-locking_get

@@ -0,0 +1,13 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.Get(ctx, tk(1)) // (v1, <nil>)
+  txn.Put(ctx, tk(3), sv(3)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000003"/0.000000003,0 @ s3 v3

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_non-locking_scan

@@ -0,0 +1,15 @@
+echo
+----
+db0.Put(ctx, tk(1), sv(1)) // @0.000000001,0 <nil>
+db0.Put(ctx, tk(1), sv(2)) // @0.000000002,0 <nil>
+db0.Put(ctx, tk(2), sv(3)) // @0.000000001,0 <nil>
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.Scan(ctx, tk(1), tk(3), 0) // (/Table/100/"0000000000000001":v1, /Table/100/"0000000000000002":v3, <nil>)
+  txn.Put(ctx, tk(3), sv(4)) // <nil>
+  return nil
+}) // @0.000000003,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000001"/0.000000002,0 @ s2 v2
+/Table/100/"0000000000000002"/0.000000001,0 @ s3 v3
+/Table/100/"0000000000000003"/0.000000003,0 @ s4 v4

pkg/kv/kvnemesis/testdata/TestValidate/weak_isolation_transaction_with_non-atomic_writes

@@ -0,0 +1,11 @@
+echo
+----
+db0.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
+  txn.SetIsoLevel(isolation.Snapshot)
+  txn.Put(ctx, tk(1), sv(1)) // <nil>
+  txn.Put(ctx, tk(2), sv(2)) // <nil>
+  return nil
+}) // @0.000000002,0 <nil>
+/Table/100/"0000000000000001"/0.000000001,0 @ s1 v1
+/Table/100/"0000000000000002"/0.000000002,0 @ s2 v2
+committed snapshot txn non-atomic timestamps: [w]/Table/100/"0000000000000001":0.000000001,0->v1@s1 [w]/Table/100/"0000000000000002":0.000000002,0->v2@s2