diff --git a/src/toolkit/Community.VisualStudio.Toolkit.Shared/MEF/SameWordHighlighterBase.cs b/src/toolkit/Community.VisualStudio.Toolkit.Shared/MEF/SameWordHighlighterBase.cs
index 69d226d..21292b1 100644
--- a/src/toolkit/Community.VisualStudio.Toolkit.Shared/MEF/SameWordHighlighterBase.cs
+++ b/src/toolkit/Community.VisualStudio.Toolkit.Shared/MEF/SameWordHighlighterBase.cs
@@ -98,10 +98,20 @@ private void CaretPositionChanged(object sender, CaretPositionChangedEventArgs e
             UpdateAtCaretPosition(e.NewPosition);
         }
 
+        private void ClearSpans()
+        {
+            lock (_syncLock)
+            {
+                _currentWord = null;
+                _wordSpans = new();
+                SnapshotSpan span = new(_buffer.CurrentSnapshot, 0, _buffer.CurrentSnapshot.Length);
+                TagsChanged?.Invoke(this, new SnapshotSpanEventArgs(span));
+            }
+        }
         private void UpdateAtCaretPosition(CaretPosition caretPosition)
         {
             SnapshotPoint? point = caretPosition.Point.GetPoint(_buffer, caretPosition.Affinity);
-
+            
             if (!point.HasValue)
             {
                 return;
@@ -117,6 +127,13 @@ private void UpdateAtCaretPosition(CaretPosition caretPosition)
                     UpdateWordAdornments(word.Value);
                 }, VsTaskRunContext.UIThreadIdlePriority).FireAndForget();
             }
+            else
+            {
+                // Clear the spans, to make sure that the highlights are 
+                // removed when we move the caret to whitespace 
+                ClearSpans();
+            }
+
         }
 
         private void UpdateWordAdornments(TextExtent word)