diff --git a/src/vs/workbench/contrib/terminal/browser/addons/commandTrackerAddon.ts b/src/vs/workbench/contrib/terminal/browser/addons/commandTrackerAddon.ts
index 3c4069967bae4..1911f12ef4c14 100644
--- a/src/vs/workbench/contrib/terminal/browser/addons/commandTrackerAddon.ts
+++ b/src/vs/workbench/contrib/terminal/browser/addons/commandTrackerAddon.ts
@@ -64,7 +64,18 @@ export class CommandTrackerAddon implements ICommandTracker, ITerminalAddon {
 		}
 
 		let markerIndex;
-		if (this._currentMarker === Boundary.Bottom) {
+		const currentLineY = Math.min(this._getLine(this._terminal, this._currentMarker), this._terminal.buffer.baseY);
+		const viewportY = this._terminal.buffer.viewportY;
+		if (!retainSelection && currentLineY !== viewportY) {
+			// The user has scrolled, find the line based on the current scroll position. This only
+			// works when not retaining selection
+			const markersBelowViewport = this._terminal.markers.filter(e => e.line >= viewportY).length;
+			if (markersBelowViewport === 0) {
+				markerIndex = -1;
+			} else {
+				markerIndex = this._terminal.markers.length - markersBelowViewport - 1;
+			}
+		} else if (this._currentMarker === Boundary.Bottom) {
 			markerIndex = this._terminal.markers.length - 1;
 		} else if (this._currentMarker === Boundary.Top) {
 			markerIndex = -1;
@@ -95,7 +106,18 @@ export class CommandTrackerAddon implements ICommandTracker, ITerminalAddon {
 		}
 
 		let markerIndex;
-		if (this._currentMarker === Boundary.Bottom) {
+		const currentLineY = Math.min(this._getLine(this._terminal, this._currentMarker), this._terminal.buffer.baseY);
+		const viewportY = this._terminal.buffer.viewportY;
+		if (!retainSelection && currentLineY !== viewportY) {
+			// The user has scrolled, find the line based on the current scroll position. This only
+			// works when not retaining selection
+			const markersAboveViewport = this._terminal.markers.filter(e => e.line <= viewportY).length;
+			if (markersAboveViewport < this._terminal.markers.length) {
+				markerIndex = markersAboveViewport;
+			} else {
+				markerIndex = this._terminal.markers.length;
+			}
+		} else if (this._currentMarker === Boundary.Bottom) {
 			markerIndex = this._terminal.markers.length;
 		} else if (this._currentMarker === Boundary.Top) {
 			markerIndex = 0;