Merge pull request cockroachdb#53755 from yuzefovich/backport20.1-53722

release-20.1: tree: fix window functions in an edge case

pkg/sql/logictest/testdata/logic_test/window

@@ -3806,3 +3806,20 @@ SELECT a, b, rank() OVER w, dense_rank() OVER w, percent_rank() OVER w, cume_dis
 0  2  2  2  1  1
 1  1  1  1  0  0.5
 1  2  2  2  1  1
+
+# Regression test for peer group number computation overflow (#53654).
+query II rowsort
+SELECT
+  max(k) OVER (w GROUPS BETWEEN 9223372036854775807 FOLLOWING AND UNBOUNDED FOLLOWING),
+  max(k) OVER (w GROUPS BETWEEN UNBOUNDED PRECEDING AND 9223372036854775807 FOLLOWING)
+FROM kv WINDOW w AS (PARTITION BY b ORDER BY v)
+----
+NULL  11
+NULL  11
+NULL  11
+NULL  8
+NULL  8
+NULL  7
+NULL  7
+NULL  7
+NULL  7

pkg/sql/sem/tree/window_funcs.go

@@ -234,7 +234,7 @@ func (wfr *WindowFrameRun) FrameStartIdx(ctx context.Context, evalCtx *EvalConte
 			offset := MustBeDInt(wfr.StartBoundOffset)
 			peerGroupNum := wfr.CurRowPeerGroupNum + int(offset)
 			lastPeerGroupNum := wfr.PeerHelper.GetLastPeerGroupNum()
-			if peerGroupNum > lastPeerGroupNum {
+			if peerGroupNum > lastPeerGroupNum || peerGroupNum < 0 {
 				// peerGroupNum is out of bounds, so we return the index of the first
 				// row after the partition.
 				return wfr.unboundedFollowing(), nil
@@ -413,7 +413,7 @@ func (wfr *WindowFrameRun) FrameEndIdx(ctx context.Context, evalCtx *EvalContext
 			offset := MustBeDInt(wfr.EndBoundOffset)
 			peerGroupNum := wfr.CurRowPeerGroupNum + int(offset)
 			lastPeerGroupNum := wfr.PeerHelper.GetLastPeerGroupNum()
-			if peerGroupNum > lastPeerGroupNum {
+			if peerGroupNum > lastPeerGroupNum || peerGroupNum < 0 {
 				// peerGroupNum is out of bounds, so we return the index of the first
 				// row after the partition.
 				return wfr.unboundedFollowing(), nil