diff --git a/packages/rrweb/src/replay/index.ts b/packages/rrweb/src/replay/index.ts
index ce4dc6d8cb..97a37f0524 100644
--- a/packages/rrweb/src/replay/index.ts
+++ b/packages/rrweb/src/replay/index.ts
@@ -300,7 +300,6 @@ export class Replayer {
 
     const timer = new Timer([], {
       speed: this.config.speed,
-      liveMode: this.config.liveMode,
     });
     this.service = createPlayerService(
       {
@@ -719,18 +718,16 @@ export class Replayer {
           this.service.send('END');
           this.emitter.emit(ReplayerEvents.Finish);
         };
+        let finish_buffer = 50; // allow for checking whether new events aren't just about to be loaded in
         if (
           event.type === EventType.IncrementalSnapshot &&
           event.data.source === IncrementalSource.MouseMove &&
           event.data.positions.length
         ) {
-          // defer finish event if the last event is a mouse move
-          setTimeout(() => {
-            finish();
-          }, Math.max(0, -event.data.positions[0].timeOffset + 50)); // Add 50 to make sure the timer would check the last mousemove event. Otherwise, the timer may be stopped by the service before checking the last event.
-        } else {
-          finish();
+          // extend finish event if the last event is a mouse move so that the timer isn't stopped by the service before checking the last event
+          finish_buffer += Math.max(0, -event.data.positions[0].timeOffset);
         }
+        setTimeout(finish, finish_buffer);
       }
 
       this.emitter.emit(ReplayerEvents.EventCast, event);
diff --git a/packages/rrweb/src/replay/machine.ts b/packages/rrweb/src/replay/machine.ts
index 9d09123f12..07d948fdca 100644
--- a/packages/rrweb/src/replay/machine.ts
+++ b/packages/rrweb/src/replay/machine.ts
@@ -223,7 +223,6 @@ export function createPlayerService(
         }),
         startLive: assign({
           baselineTime: (ctx, event) => {
-            ctx.timer.toggleLiveMode(true);
             ctx.timer.start();
             if (event.type === 'TO_LIVE' && event.payload.baselineTime) {
               return event.payload.baselineTime;
diff --git a/packages/rrweb/src/replay/timer.ts b/packages/rrweb/src/replay/timer.ts
index e25e6a4647..0d18c3f870 100644
--- a/packages/rrweb/src/replay/timer.ts
+++ b/packages/rrweb/src/replay/timer.ts
@@ -10,64 +10,71 @@ export class Timer {
   public speed: number;
 
   private actions: actionWithDelay[];
-  private raf: number | null = null;
-  private liveMode: boolean;
+  private raf: number | true | null = null;
+  private lastTimestamp: number;
 
   constructor(
     actions: actionWithDelay[] = [],
     config: {
       speed: number;
-      liveMode: boolean;
     },
   ) {
     this.actions = actions;
     this.speed = config.speed;
-    this.liveMode = config.liveMode;
   }
   /**
    * Add an action, possibly after the timer starts.
    */
   public addAction(action: actionWithDelay) {
+    const rafWasActive = this.raf === true;
     if (
       !this.actions.length ||
       this.actions[this.actions.length - 1].delay <= action.delay
     ) {
       // 'fast track'
       this.actions.push(action);
-      return;
+    } else {
+      // binary search - events can arrive out of order in a realtime context
+      const index = this.findActionIndex(action);
+      this.actions.splice(index, 0, action);
+    }
+    if (rafWasActive) {
+      this.raf = requestAnimationFrame(this.rafCheck.bind(this));
     }
-    // binary search - events can arrive out of order in a realtime context
-    const index = this.findActionIndex(action);
-    this.actions.splice(index, 0, action);
   }
 
   public start() {
     this.timeOffset = 0;
-    let lastTimestamp = performance.now();
-    const check = () => {
-      const time = performance.now();
-      this.timeOffset += (time - lastTimestamp) * this.speed;
-      lastTimestamp = time;
-      while (this.actions.length) {
-        const action = this.actions[0];
+    this.lastTimestamp = performance.now();
+    this.raf = requestAnimationFrame(this.rafCheck.bind(this));
+  }
 
-        if (this.timeOffset >= action.delay) {
-          this.actions.shift();
-          action.doAction();
-        } else {
-          break;
-        }
-      }
-      if (this.actions.length > 0 || this.liveMode) {
-        this.raf = requestAnimationFrame(check);
+  private rafCheck() {
+    const time = performance.now();
+    this.timeOffset += (time - this.lastTimestamp) * this.speed;
+    this.lastTimestamp = time;
+    while (this.actions.length) {
+      const action = this.actions[0];
+
+      if (this.timeOffset >= action.delay) {
+        this.actions.shift();
+        action.doAction();
+      } else {
+        break;
       }
-    };
-    this.raf = requestAnimationFrame(check);
+    }
+    if (this.actions.length > 0) {
+      this.raf = requestAnimationFrame(this.rafCheck.bind(this));
+    } else {
+      this.raf = true; // was active
+    }
   }
 
   public clear() {
     if (this.raf) {
-      cancelAnimationFrame(this.raf);
+      if (this.raf !== true) {
+        cancelAnimationFrame(this.raf);
+      }
       this.raf = null;
     }
     this.actions.length = 0;
@@ -77,10 +84,6 @@ export class Timer {
     this.speed = speed;
   }
 
-  public toggleLiveMode(mode: boolean) {
-    this.liveMode = mode;
-  }
-
   public isActive() {
     return this.raf !== null;
   }