Merge #51079

51079: kv/gc: don't continue iterating and spamming logs when deadline exceeded r=nvanbenschoten a=nvanbenschoten

Before this change, we'd see logs like:
```
W200706 11:23:10.682177 15143568003 kv/kvserver/gc/gc.go:337  [n118,gc,s118,r6153/1:/Table/57/1/"user1216{19…-44…}] failed to GC a batch of keys: result is ambiguous (context deadline exceeded)
W200706 11:23:10.685251 15143568003 kv/kvserver/gc/gc.go:337  [n118,gc,s118,r6153/1:/Table/57/1/"user1216{19…-44…}] failed to GC a batch of keys: aborted during Replica.Send: context deadline exceeded
W200706 11:23:10.688667 15143568003 kv/kvserver/gc/gc.go:337  [n118,gc,s118,r6153/1:/Table/57/1/"user1216{19…-44…}] failed to GC a batch of keys: aborted during Replica.Send: context deadline exceeded
W200706 11:23:10.691760 15143568003 kv/kvserver/gc/gc.go:337  [n118,gc,s118,r6153/1:/Table/57/1/"user1216{19…-44…}] failed to GC a batch of keys: aborted during Replica.Send: context deadline exceeded
...
```

if a GC attempt hit its deadline. Now, we terminate iteration early and do our best to stop running GC once the context is canceled / exceeding its deadline.

Co-authored-by: Nathan VanBenschoten <[email protected]>

pkg/kv/kvserver/gc/gc.go

@@ -191,12 +191,20 @@ func Run(
 
 	// Process local range key entries (txn records, queue last processed times).
 	if err := processLocalKeyRange(ctx, snap, desc, txnExp, &info, cleanupTxnIntentsAsyncFn, gcer); err != nil {
+		if errors.Is(err, ctx.Err()) {
+			return Info{}, err
+		}
 		log.Warningf(ctx, "while gc'ing local key range: %s", err)
 	}
 
 	// Clean up the AbortSpan.
 	log.Event(ctx, "processing AbortSpan")
-	processAbortSpan(ctx, snap, desc.RangeID, txnExp, &info, gcer)
+	if err := processAbortSpan(ctx, snap, desc.RangeID, txnExp, &info, gcer); err != nil {
+		if errors.Is(err, ctx.Err()) {
+			return Info{}, err
+		}
+		log.Warningf(ctx, "while gc'ing abort span: %s", err)
+	}
 
 	log.Eventf(ctx, "GC'ed keys; stats %+v", info)
 
@@ -329,6 +337,9 @@ func processReplicatedKeyRange(
 		}
 		if shouldSendBatch {
 			if err := gcer.GC(ctx, batchGCKeys); err != nil {
+				if errors.Is(err, ctx.Err()) {
+					return err
+				}
 				// Even though we are batching the GC process, it's
 				// safe to continue because we bumped the GC
 				// thresholds. We may leave some inconsistent history
@@ -491,23 +502,20 @@ func processAbortSpan(
 	threshold hlc.Timestamp,
 	info *Info,
 	gcer PureGCer,
-) {
+) error {
 	b := makeBatchingInlineGCer(gcer, func(err error) {
 		log.Warningf(ctx, "unable to GC from abort span: %s", err)
 	})
 	defer b.Flush(ctx)
 	abortSpan := abortspan.New(rangeID)
-	err := abortSpan.Iterate(ctx, snap, func(key roachpb.Key, v roachpb.AbortSpanEntry) error {
+	return abortSpan.Iterate(ctx, snap, func(key roachpb.Key, v roachpb.AbortSpanEntry) error {
 		info.AbortSpanTotal++
 		if v.Timestamp.Less(threshold) {
 			info.AbortSpanGCNum++
 			b.FlushingAdd(ctx, key)
 		}
 		return nil
 	})
-	if err != nil {
-		log.Warningf(ctx, "%v", err)
-	}
 }
 
 // batchingInlineGCer is a helper to paginate the GC of inline (i.e. zero

pkg/kv/kvserver/gc/gc_old_test.go

@@ -222,7 +222,9 @@ func runGCOld(
 
 	// Clean up the AbortSpan.
 	log.Event(ctx, "processing AbortSpan")
-	processAbortSpan(ctx, snap, desc.RangeID, txnExp, &info, gcer)
+	if err := processAbortSpan(ctx, snap, desc.RangeID, txnExp, &info, gcer); err != nil {
+		log.Warningf(ctx, "while gc'ing abort span: %s", err)
+	}
 
 	log.Eventf(ctx, "GC'ed keys; stats %+v", info)