diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh
index 32ede6ad817980..37c1c5d227b403 100755
--- a/git-gui/git-gui.sh
+++ b/git-gui/git-gui.sh
@@ -2513,6 +2513,19 @@ proc toggle_or_diff {mode w args} {
 		set pos [split [$w index @$x,$y] .]
 		foreach {lno col} $pos break
 	} else {
+		if {$mode eq "toggle"} {
+			if {$w eq $ui_workdir} {
+				do_add_selection
+				set last_clicked {}
+				return
+			}
+			if {$w eq $ui_index} {
+				do_unstage_selection
+				set last_clicked {}
+				return
+			}
+		}
+
 		if {$last_clicked ne {}} {
 			set lno [lindex $last_clicked 1]
 		} else {