diff --git a/whisper/transcribe.py b/whisper/transcribe.py
index cba59ecfa..ff73a5530 100644
--- a/whisper/transcribe.py
+++ b/whisper/transcribe.py
@@ -312,10 +312,6 @@ def new_segment(
                 )
                 seek += segment_size
 
-            if not condition_on_previous_text or result.temperature > 0.5:
-                # do not feed the prompt tokens if a high temperature was used
-                prompt_reset_since = len(all_tokens)
-
             if word_timestamps:
                 add_word_timestamps(
                     segments=current_segments,
@@ -361,6 +357,10 @@ def new_segment(
                 [token for segment in current_segments for token in segment["tokens"]]
             )
 
+            if not condition_on_previous_text or result.temperature > 0.5:
+                # do not feed the prompt tokens if a high temperature was used
+                prompt_reset_since = len(all_tokens)
+
             # update progress bar
             pbar.update(min(content_frames, seek) - previous_seek)