fix: exclude bot traffic from page views and opportunities ctr calculations

packages/spacecat-shared-rum-api-client/src/common/rum-bundler-client.js

@@ -22,7 +22,11 @@ const ONE_DAY = ONE_HOUR * HOURS_IN_DAY;
 
 const CHUNK_SIZE = 31;
 
-function filterBundles(checkpoints = []) {
+function isBotTraffic(bundle) {
+  return bundle?.userAgent?.includes('bot');
+}
+
+function filterEvents(checkpoints = []) {
   return (bundle) => {
     if (checkpoints.length > 0) {
       const events = bundle.events.filter((event) => checkpoints.includes(event.checkpoint));
@@ -160,6 +164,7 @@ async function fetchBundles(opts = {}) {
     interval = 7,
     granularity = GRANULARITY.DAILY,
     checkpoints = [],
+    filterBotTraffic = true,
   } = opts;
 
   if (!hasText(domain) || !hasText(domainkey)) {
@@ -185,7 +190,11 @@ async function fetchBundles(opts = {}) {
   for (const chunk of chunks) {
     const responses = await Promise.all(chunk.map((url) => fetch(url)));
     const bundles = await Promise.all(responses.map((response) => response.json()));
-    result.push(...bundles.flatMap((b) => b.rumBundles.map(filterBundles(checkpoints))));
+    result.push(...bundles.flatMap(
+      (b) => b.rumBundles.filter(
+        (bundle) => !filterBotTraffic || !isBotTraffic(bundle),
+      ).map(filterEvents(checkpoints)),
+    ));
   }
   return mergeBundlesWithSameId(result);
 }

packages/spacecat-shared-rum-api-client/test/common/rum-bundler-client.test.js

@@ -24,7 +24,7 @@
 describe('Rum bundler client', () => {
   let sandbox;
 
   before('setup', function () {
     const mockDate = '2024-06-02T12:30:01.124Z';
     sandbox = sinon.createSandbox();
     this.clock = sandbox.useFakeTimers({
@@ -32,7 +32,7 @@
     });
   });
 
   after('clean', function () {
     this.clock.uninstall();
   });
 
@@ -72,9 +72,69 @@
     const containsCorrectData = result.every((item) => item.events.length === 1 && item.events[0].checkpoint === 'good');
 
     expect(result.length).to.equal(dates.length);
+    // eslint-disable-next-line no-unused-expressions
     expect(containsCorrectData).to.be.true;
   });
 
+  it('should filter bot traffic by default', async () => {
+    const domain = 'some-domain.com';
+    const domainkey = 'testkey';
+    const granularity = 'HOURLY';
+    const interval = 7;
+    const allCheckpoints = ['good', 'bad'];
+
+    const dates = generateHourlyDates(7);
+    const rumBundles = generateRumBundles(dates, allCheckpoints);
+    // make the first bundle a bot traffic
+    rumBundles[Object.keys(rumBundles)[0]].rumBundles[0].userAgent = 'bot';
+
+    for (const date of dates) {
+      nock(BASE_URL)
+        .get(`/bundles/${domain}/${date[0]}/${date[1]}/${date[2]}/${date[3]}?domainkey=${domainkey}`)
+        .reply(200, rumBundles[date.join()]);
+    }
+
+    const result = await fetchBundles({
+      domain,
+      domainkey,
+      granularity,
+      interval,
+      checkpoints: ['good'],
+    });
+
+    expect(result.length).to.equal(dates.length - 1);
+  });
+
+  it('should not filter bot traffic when disabled', async () => {
+    const domain = 'some-domain.com';
+    const domainkey = 'testkey';
+    const granularity = 'HOURLY';
+    const interval = 7;
+    const allCheckpoints = ['good', 'bad'];
+
+    const dates = generateHourlyDates(7);
+    const rumBundles = generateRumBundles(dates, allCheckpoints);
+    // make the first bundle a bot traffic
+    rumBundles[Object.keys(rumBundles)[0]].rumBundles[0].userAgent = 'bot';
+
+    for (const date of dates) {
+      nock(BASE_URL)
+        .get(`/bundles/${domain}/${date[0]}/${date[1]}/${date[2]}/${date[3]}?domainkey=${domainkey}`)
+        .reply(200, rumBundles[date.join()]);
+    }
+
+    const result = await fetchBundles({
+      domain,
+      domainkey,
+      granularity,
+      interval,
+      checkpoints: ['good'],
+      filterBotTraffic: false,
+    });
+
+    expect(result.length).to.equal(dates.length);
+  });
+
   it('should fetch correct daily bundles and filter by checkpoint', async () => {
     const domain = 'some-domain.com';
     const domainkey = 'testkey';
@@ -102,6 +162,7 @@
     const containsCorrectData = result.every((item) => item.events.length === 1 && item.events[0].checkpoint === 'good');
 
     expect(result.length).to.equal(dates.length);
+    // eslint-disable-next-line no-unused-expressions
     expect(containsCorrectData).to.be.true;
   });
 
@@ -132,6 +193,7 @@
     const containsCorrectData = result.every((item) => item.events.length === 2);
 
     expect(result.length).to.equal(dates.length);
+    // eslint-disable-next-line no-unused-expressions
     expect(containsCorrectData).to.be.true;
   });
 });

packages/spacecat-shared-rum-api-client/test/index.test.js

@@ -52,7 +52,7 @@ describe('RUMAPIClient', () => {
       interval: 0,
     };
     const result = await rumApiClient.query('404', opts);
-
+    // eslint-disable-next-line no-unused-expressions
     expect(result).to.be.empty;
   });