fix: Fix deferred Session Replay payloads (#868)

src/features/metrics/aggregate/index.js

@@ -102,7 +102,7 @@ export class Aggregate extends AggregateBase {
 
     // [Temporary] Report restores from BFCache to NR1 while feature flag is in place in lieu of sending pageshow events.
     windowAddEventListener('pageshow', (evt) => {
-      if (evt.persisted) { this.storeSupportabilityMetrics('Generic/BFCache/PageRestored') }
+      if (evt?.persisted) { this.storeSupportabilityMetrics('Generic/BFCache/PageRestored') }
     })
   }
 

src/features/session_replay/aggregate/index.component-test.js

@@ -273,10 +273,11 @@ describe('Session Replay', () => {
       }))
 
       setConfiguration(agentIdentifier, { ...init })
-      sr.shouldCompress = false
       sr.ee.emit('rumresp-sr', [true])
       await wait(1)
 
+      sr.gzipper = undefined
+
       const [harvestContents] = sr.prepareHarvest()
       expect(harvestContents.qs).toMatchObject({
         protocol_version: '0',
@@ -289,10 +290,10 @@ describe('Session Replay', () => {
 
     test('Clears the event buffer when staged for harvesting', async () => {
       setConfiguration(agentIdentifier, { ...init })
-      sr.shouldCompress = false
       sr.ee.emit('rumresp-sr', [true])
       await wait(1)
 
+      sr.gzipper = undefined
       sr.prepareHarvest()
       expect(sr.recorder.getEvents().events.length).toEqual(0)
     })
@@ -311,9 +312,12 @@ describe('Session Replay', () => {
     })
 
     test('Aborts if exceeds total limit', async () => {
+      jest.doMock('fflate', () => ({
+        __esModule: true,
+        gzipSync: jest.fn().mockImplementation(() => { throw new Error() })
+      }))
       const spy = jest.spyOn(sr.scheduler.harvest, '_send')
       setConfiguration(agentIdentifier, { ...init })
-      sr.shouldCompress = false
       sr.recorder = new Recorder(sr)
       Array.from({ length: 100000 }).forEach(() => sr.recorder.currentBufferTarget.add({ test: 1 })) //  fill the events array with tons of events
       sr.recorder.currentBufferTarget.payloadBytesEstimation = sr.recorder.currentBufferTarget.events.join('').length

src/features/session_replay/aggregate/index.js

@@ -28,8 +28,6 @@ import { MODE, SESSION_EVENTS, SESSION_EVENT_TYPES } from '../../../common/sessi
 import { stringify } from '../../../common/util/stringify'
 import { stylesheetEvaluator } from '../shared/stylesheet-evaluator'
 
-let gzipper, u8
-
 export class Aggregate extends AggregateBase {
   static featureName = FEATURE_NAME
   // pass the recorder into the aggregator
@@ -41,8 +39,10 @@ export class Aggregate extends AggregateBase {
     this.initialized = false
     /** Set once the feature has been "aborted" to prevent other side-effects from continuing */
     this.blocked = false
-    /** can shut off efforts to compress the data */
-    this.shouldCompress = true
+    /** populated with the gzipper lib async */
+    this.gzipper = undefined
+    /** populated with the u8 string lib async */
+    this.u8 = undefined
     /** the mode to start in.  Defaults to off */
     const { session } = getRuntime(this.agentIdentifier)
     this.mode = session.state.sessionReplayMode || MODE.OFF
@@ -91,6 +91,12 @@ export class Aggregate extends AggregateBase {
         raw: true
       }, this)
 
+      if (this.recorder?.getEvents().type === 'preloaded') {
+        this.prepUtils().then(() => {
+          this.scheduler.runHarvest()
+        })
+      }
+
       registerHandler('recordReplay', () => {
         // if it has aborted or BCS returned bad entitlements, do not allow
         if (this.blocked || !this.entitled) return
@@ -197,18 +203,22 @@ export class Aggregate extends AggregateBase {
       this.scheduler.startTimer(this.harvestTimeSeconds)
     }
 
+    await this.prepUtils()
+
+    if (!this.recorder.recording) this.recorder.startRecording()
+
+    this.syncWithSessionManager({ sessionReplayMode: this.mode })
+  }
+
+  async prepUtils () {
     try {
       // Do not change the webpackChunkName or it will break the webpack nrba-chunking plugin
       const { gzipSync, strToU8 } = await import(/* webpackChunkName: "compressor" */'fflate')
-      gzipper = gzipSync
-      u8 = strToU8
+      this.gzipper = gzipSync
+      this.u8 = strToU8
     } catch (err) {
       // compressor failed to load, but we can still record without compression as a last ditch effort
-      this.shouldCompress = false
     }
-    if (!this.recorder.recording) this.recorder.startRecording()
-
-    this.syncWithSessionManager({ sessionReplayMode: this.mode })
   }
 
   prepareHarvest () {
@@ -224,8 +234,8 @@ export class Aggregate extends AggregateBase {
     }
 
     let len = 0
-    if (this.shouldCompress) {
-      payload.body = gzipper(u8(`[${payload.body.map(e => e.__serialized).join(',')}]`))
+    if (!!this.gzipper && !!this.u8) {
+      payload.body = this.gzipper(this.u8(`[${payload.body.map(e => e.__serialized).join(',')}]`))
       len = payload.body.length
       this.scheduler.opts.gzip = true
     } else {
@@ -276,7 +286,7 @@ export class Aggregate extends AggregateBase {
 
     const firstEventTimestamp = events[0]?.timestamp // from rrweb node
     const lastEventTimestamp = events[events.length - 1]?.timestamp // from rrweb node
-    const firstTimestamp = firstEventTimestamp || recorderEvents.cycleTimestamp
+    const firstTimestamp = firstEventTimestamp || recorderEvents.cycleTimestamp // from rrweb node || from when the harvest cycle started
     const lastTimestamp = lastEventTimestamp || agentOffset + relativeNow
 
     return {
@@ -288,7 +298,7 @@ export class Aggregate extends AggregateBase {
         attributes: encodeObj({
           // this section of attributes must be controllable and stay below the query param padding limit -- see QUERY_PARAM_PADDING
           // if not, data could be lost to truncation at time of sending, potentially breaking parsing / API behavior in NR1
-          ...(this.shouldCompress && { content_encoding: 'gzip' }),
+          ...(!!this.gzipper && !!this.u8 && { content_encoding: 'gzip' }),
           'replay.firstTimestamp': firstTimestamp,
           'replay.firstTimestampOffset': firstTimestamp - agentOffset,
           'replay.lastTimestamp': lastTimestamp,

src/features/session_replay/shared/recorder.js

@@ -184,7 +184,7 @@ export class Recorder {
    * https://staging.onenr.io/037jbJWxbjy
    * */
   estimateCompression (data) {
-    if (this.shouldCompress) return data * AVG_COMPRESSION
+    if (!!this.parent.gzipper && !!this.parent.u8) return data * AVG_COMPRESSION
     return data
   }
 }

src/features/session_replay/shared/stylesheet-evaluator.js

@@ -55,27 +55,32 @@ class StylesheetEvaluator {
  * @returns {Promise}
  */
   async #fetchAndOverride (target, href) {
-    const stylesheetContents = await originals.FETCH.bind(window)(href)
-    if (!stylesheetContents.ok) {
-      this.failedToFix = true
-      return
-    }
-    const stylesheetText = await stylesheetContents.text()
     try {
-      const cssSheet = new CSSStyleSheet()
-      await cssSheet.replace(stylesheetText)
-      Object.defineProperty(target, 'cssRules', {
-        get () { return cssSheet.cssRules }
-      })
-      Object.defineProperty(target, 'rules', {
-        get () { return cssSheet.rules }
-      })
-    } catch (err) {
+      const stylesheetContents = await originals.FETCH.bind(window)(href)
+      if (!stylesheetContents.ok) {
+        this.failedToFix = true
+        return
+      }
+      const stylesheetText = await stylesheetContents.text()
+      try {
+        const cssSheet = new CSSStyleSheet()
+        await cssSheet.replace(stylesheetText)
+        Object.defineProperty(target, 'cssRules', {
+          get () { return cssSheet.cssRules }
+        })
+        Object.defineProperty(target, 'rules', {
+          get () { return cssSheet.rules }
+        })
+      } catch (err) {
       // cant make new dynamic stylesheets, browser likely doesn't support `.replace()`...
       // this is appended in prep of forking rrweb
-      Object.defineProperty(target, 'cssText', {
-        get () { return stylesheetText }
-      })
+        Object.defineProperty(target, 'cssText', {
+          get () { return stylesheetText }
+        })
+        this.failedToFix = true
+      }
+    } catch (err) {
+    // failed to fetch
       this.failedToFix = true
     }
   }

tests/specs/session-replay/session-pages.e2e.js

@@ -1,4 +1,3 @@
-import { testRumRequest } from '../../../tools/testing-server/utils/expect-tests.js'
 import { config, getSR, testExpectedReplay } from './helpers'
 import { supportsMultipleTabs, notIE, notSafari } from '../../../tools/browser-matcher/common-matchers.mjs'
 
@@ -39,25 +38,14 @@ describe.withBrowsersMatching(notIE)('Session Replay Across Pages', () => {
     const { request: page1Contents } = await browser.testHandle.expectBlob(10000)
     testExpectedReplay({ data: page1Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: true })
 
-    await browser.testHandle.scheduleReply('bamServer', {
-      test: testRumRequest,
-      body: JSON.stringify({
-        stn: 1,
-        err: 1,
-        ins: 1,
-        cap: 1,
-        spa: 1,
-        loaded: 1,
-        sr: 1
-      })
-    })
-
     await browser.enableSessionReplay()
 
-    await browser.url(await browser.testHandle.assetURL('instrumented.html', config()))
-      .then(() => browser.waitForAgentLoad())
+    const [{ request: page2Contents }] = await Promise.all([
+      browser.testHandle.expectBlob(15000),
+      browser.url(await browser.testHandle.assetURL('instrumented.html', config()))
+        .then(() => browser.waitForAgentLoad())
+    ])
 
-    const { request: page2Contents } = await browser.testHandle.expectBlob(10000)
     testExpectedReplay({ data: page2Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: false })
   })
 
@@ -68,21 +56,26 @@ describe.withBrowsersMatching(notIE)('Session Replay Across Pages', () => {
 
     const { request: page1Contents } = await browser.testHandle.expectBlob(10000)
     const { localStorage } = await browser.getAgentSessionInfo()
-
     testExpectedReplay({ data: page1Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: true })
 
-    const newTab = await browser.createWindow('tab')
-    await browser.switchToWindow(newTab.handle)
-    await browser.enableSessionReplay()
-    await browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
-      .then(() => browser.waitForAgentLoad())
-
-    const { request: page2Contents } = await browser.testHandle.expectBlob(10000)
+    /** This should fire when the tab changes, it's easier to stage it this way before hand, and allows for the super early staging for the next expect */
+    browser.testHandle.expectBlob(15000).then(({ request: page1UnloadContents }) => {
+      testExpectedReplay({ data: page1UnloadContents, session: localStorage.value, hasError: false, hasMeta: false, hasSnapshot: false, isFirstChunk: false })
+    })
 
-    testExpectedReplay({ data: page2Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: false })
+    /** This is scoped out this way to guarantee we have it staged in time since preload can harvest super early, sometimes earlier than wdio can expect normally */
+    /** see next `testExpectedReplay` */
+    browser.testHandle.expectBlob(15000).then(async ({ request: page2Contents }) => {
+      testExpectedReplay({ data: page2Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: false })
+      // await browser.closeWindow()
+      // await browser.switchToWindow((await browser.getWindowHandles())[0])
+    })
 
-    await browser.closeWindow()
-    await browser.switchToWindow((await browser.getWindowHandles())[0])
+    await browser.enableSessionReplay()
+    const newTab = await browser.createWindow('tab')
+    await browser.switchToWindow(newTab.handle)
+    await browser.url(await browser.testHandle.assetURL('instrumented.html', config()))
+      .then(() => browser.waitForSessionReplayRecording())
   })
 
   it('should not record across navigations if not active', async () => {
@@ -115,15 +108,22 @@ describe.withBrowsersMatching(notIE)('Session Replay Across Pages', () => {
 
     testExpectedReplay({ data: page1Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: true })
 
-    const newTab = await browser.createWindow('tab')
-    await browser.switchToWindow(newTab.handle)
-    await browser.enableSessionReplay()
-    await browser.url(await browser.testHandle.assetURL('rrweb-instrumented.html', config()))
-      .then(() => browser.waitForAgentLoad())
+    /** This should fire when the tab changes, it's easier to stage it this way before hand, and allows for the super early staging for the next expect */
+    browser.testHandle.expectBlob(15000).then(({ request: page1UnloadContents }) => {
+      testExpectedReplay({ data: page1UnloadContents, session: localStorage.value, hasError: false, hasMeta: false, hasSnapshot: false, isFirstChunk: false })
+    })
 
-    const { request: page2Contents } = await browser.testHandle.expectBlob(10000)
+    /** This is scoped out this way to guarantee we have it staged in time since preload can harvest super early, sometimes earlier than wdio can expect normally */
+    /** see next `testExpectedReplay` */
+    browser.testHandle.expectBlob(15000).then(async ({ request: page2Contents }) => {
+      testExpectedReplay({ data: page2Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: false })
+    })
 
-    testExpectedReplay({ data: page2Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: false })
+    await browser.enableSessionReplay()
+    const newTab = await browser.createWindow('tab')
+    await browser.switchToWindow(newTab.handle)
+    await browser.url(await browser.testHandle.assetURL('instrumented.html', config()))
+      .then(() => browser.waitForSessionReplayRecording())
 
     const page2Blocked = await browser.execute(function () {
       try {
@@ -134,9 +134,6 @@ describe.withBrowsersMatching(notIE)('Session Replay Across Pages', () => {
         return false
       }
     })
-    await browser.closeWindow()
-    await browser.switchToWindow((await browser.getWindowHandles())[0])
-
     expect(page2Blocked).toEqual(true)
     await expect(getSR()).resolves.toEqual(expect.objectContaining({
       events: [],

tests/specs/session-replay/session-state.e2e.js

@@ -1,5 +1,5 @@
 import { supportsMultipleTabs, notIE } from '../../../tools/browser-matcher/common-matchers.mjs'
-import { RRWEB_EVENT_TYPES, config, getSR, MODE } from './helpers.js'
+import { RRWEB_EVENT_TYPES, config, MODE, testExpectedReplay } from './helpers.js'
 
 describe.withBrowsersMatching(notIE)('session manager state behavior', () => {
   beforeEach(async () => {
@@ -68,30 +68,31 @@ describe.withBrowsersMatching(notIE)('session manager state behavior', () => {
 
   describe('When session resumes', () => {
     it.withBrowsersMatching(supportsMultipleTabs)('should take a full snapshot and continue recording', async () => {
-      await browser.url(await browser.testHandle.assetURL('instrumented.html', config()))
-        .then(() => browser.waitForSessionReplayRecording())
-
-      const { events: currentPayload } = await getSR()
+      const [{ request: payload }] = await Promise.all([
+        browser.testHandle.expectBlob(15000),
+        browser.url(await browser.testHandle.assetURL('instrumented.html', config()))
+          .then(() => browser.waitForAgentLoad())
+      ])
 
-      expect(currentPayload.length).toBeGreaterThan(0)
+      expect(payload.body.length).toBeGreaterThan(0)
       // type 2 payloads are snapshots
-      expect(currentPayload.filter(x => x.type === RRWEB_EVENT_TYPES.FullSnapshot).length).toEqual(1)
-
+      expect(payload.body.filter(x => x.type === RRWEB_EVENT_TYPES.FullSnapshot).length).toEqual(1)
+
+      /** This should fire when the tab changes, it's easier to stage it this way before hand, and allows for the super early staging for the next expect */
+      browser.testHandle.expectBlob(15000).then(({ request: page1UnloadContents }) => {
+        testExpectedReplay({ data: page1UnloadContents, session: localStorage.value, hasError: false, hasMeta: false, hasSnapshot: false, isFirstChunk: false })
+      })
+
+      /** This is scoped out this way to guarantee we have it staged in time since preload can harvest super early, sometimes earlier than wdio can expect normally */
+      /** see next `testExpectedReplay` */
+      browser.testHandle.expectBlob(15000).then(async ({ request: page2Contents }) => {
+        testExpectedReplay({ data: page2Contents, session: localStorage.value, hasError: false, hasMeta: true, hasSnapshot: true, isFirstChunk: false })
+      })
       const newTab = await browser.createWindow('tab')
       await browser.switchToWindow(newTab.handle)
       await browser.enableSessionReplay()
       await browser.url(await browser.testHandle.assetURL('instrumented.html', config()))
         .then(() => browser.waitForSessionReplayRecording())
-
-      const { events: resumedPayload } = await getSR()
-
-      // payload was harvested, new vis change should trigger a new recording which includes a new full snapshot
-      expect(resumedPayload.length).toBeGreaterThan(0)
-      // type 2 payloads are snapshots
-      expect(resumedPayload.filter(x => x.type === RRWEB_EVENT_TYPES.FullSnapshot).length).toEqual(1)
-
-      await browser.closeWindow()
-      await browser.switchToWindow((await browser.getWindowHandles())[0])
     })
   })
 

tools/wdio/plugins/custom-commands.mjs

@@ -144,13 +144,20 @@ export default class CustomCommands {
         () => browser.execute(function () {
           try {
             var initializedAgent = Object.values(newrelic.initializedAgents)[0]
-            return !!(initializedAgent &&
+            return !!(
+              (initializedAgent &&
+              initializedAgent.features &&
+              initializedAgent.features.session_replay &&
+              initializedAgent.features.session_replay.recorder &&
+              initializedAgent.features.session_replay.recorder.recording) ||
+              (initializedAgent &&
               initializedAgent.features &&
               initializedAgent.features.session_replay &&
               initializedAgent.features.session_replay.featAggregate &&
               initializedAgent.features.session_replay.featAggregate.initialized &&
               initializedAgent.features.session_replay.featAggregate.recorder &&
               initializedAgent.features.session_replay.featAggregate.recorder.recording)
+            )
           } catch (err) {
             console.error(err)
             return false