Feat: recursive dom profiler (#96)

* feat(profiler): run browserstack locally
* fix(profiler): chrome unique datadir

.github/workflows/lint-and-test.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, windows-latest, ubuntu-latest]
+        os: [macos-13, windows-latest, ubuntu-latest]
         node-version: [18, 20]
         include:
           - node-version: 20.x
@@ -91,12 +91,13 @@ jobs:
         run: NODE_OPTIONS=--max-old-space-size=4096 yarn lint
 
       - name: Run tests
-        run: yarn jest --testTimeout=60000 --maxWorkers=2
+        run: yarn jest --testTimeout=60000 --maxWorkers=1
         working-directory: ./build
         env:
           ULX_DATA_DIR: .data-test
           NODE_ENV: test
           ULX_DEFAULT_BROWSER_ID: ${{ matrix.browser }}
+          NODE_OPTIONS: --max-old-space-size=4096
 
       - name: 'Tar files'
         if: ${{ failure() }}

agent/main/package.json

@@ -4,7 +4,7 @@
   "description": "Fully programmable Devtools Protocol based browser",
   "main": "index.js",
   "dependencies": {
-    "@ulixee/chrome-121-0": "^6167.86.8",
+    "@ulixee/chrome-124-0": "^6367.208.10",
     "@ulixee/chrome-app": "^1.0.3",
     "@ulixee/commons": "2.0.0-alpha.28",
     "@ulixee/js-path": "2.0.0-alpha.28",

agent/main/test/mitm.test.ts

@@ -32,13 +32,17 @@ beforeEach(async () => {
 afterAll(Helpers.afterAll);
 afterEach(Helpers.afterEach);
 
-function mitmCalls(thisTest: number, opts?: { noGoto?: boolean; noFavicon?: boolean }): number {
-  // goto and favicon
-  let calls = 2;
-  if (opts?.noFavicon) calls -= 1;
-  if (opts?.noGoto) calls -= 1;
-
-  return thisTest + calls;
+function mitmCalls(expectedCalls: number): number {
+  // Takes care of removing favicon from call mitm calls if needed.
+  // During testing favicon migth or migth not be called depending on
+  // speed of setup. To prevent flaky tests we ignore this.
+  const hasCalledFavicon = mocks.MitmRequestContext.create.mock.results.some(
+    result => result.value.url.href.includes('favicon'),
+  );
+  if (hasCalledFavicon) {
+    return expectedCalls + 1;
+  }
+  return expectedCalls;
 }
 
 test('should send a Host header to secure http1 Chrome requests', async () => {
@@ -99,7 +103,7 @@ xhr.send('<person><name>DLF</name></person>');
   await expect(corsPromise).resolves.toBeTruthy();
   await expect(postPromise).resolves.toBeTruthy();
 
-  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(2));
+  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(3));
   const results = mocks.MitmRequestContext.create.mock.results.filter(
     result => !result.value.url.href.includes('favicon'),
   );
@@ -144,7 +148,7 @@ myWorker.postMessage('send');
   await page.goto(`${koa.baseUrl}/testWorker`);
   await page.mainFrame.waitForLoad({ loadStatus: 'PaintingStable' });
   await expect(serviceXhr).resolves.toBe('FromWorker');
-  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(2));
+  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(3));
 });
 
 test('should proxy requests from shared workers', async () => {
@@ -191,7 +195,7 @@ sharedWorker.port.addEventListener('message', message => {
   await page.goto(`${server.baseUrl}/testSharedWorker`);
   await page.mainFrame.waitForLoad({ loadStatus: 'PaintingStable' });
   await expect(xhrResolvable.promise).resolves.toBe('FromSharedWorker');
-  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(2));
+  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(3));
 });
 
 test('should not see proxy headers in a service worker', async () => {
@@ -295,7 +299,7 @@ window.addEventListener('load', function() {
   await expect(originalHeaders['proxy-authorization']).not.toBeTruthy();
   await expect(headersFromWorker['proxy-authorization']).not.toBeTruthy();
   await expect(originalHeaders['user-agent']).toBe(headersFromWorker['user-agent']);
-  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(3));
+  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(4));
 });
 
 test('should proxy iframe requests', async () => {
@@ -326,8 +330,7 @@ This is the main body
   });
   await page.goto(`${koa.baseUrl}/iframe-test`);
   await page.waitForLoad(LocationStatus.AllContentLoaded);
-  // TODO why doesn't this load favicon?
-  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(3, { noFavicon: true }));
+  expect(mocks.MitmRequestContext.create).toHaveBeenCalledTimes(mitmCalls(4));
   const urls = mocks.MitmRequestContext.create.mock.results.map(x => x.value.url.href);
   expect(urls).toEqual([
     expect.stringMatching(/http:\/\/localhost:\d+\/iframe-test/),

agent/main/test/navigation.test.ts

@@ -44,6 +44,9 @@ describe.each([
   ['with Mitm', true],
   ['withoutMitm', false],
 ])('basic Navigation tests %s', (key, enableMitm) => {
+  if (!enableMitm) {
+    jest.retryTimes(3);
+  }
   beforeAll(async () => {
     koaServer = await Helpers.runKoaServer();
   });
@@ -548,6 +551,7 @@ history.pushState({}, '', '/inpagenav/1');
     });
 
     await page.goto(`${koaServer.baseUrl}/newPagePrompt`);
+    const popupPagePromise = waitForPopup(page);
     await page.interact([
       {
         command: InteractionCommand.click,
@@ -558,7 +562,7 @@ history.pushState({}, '', '/inpagenav/1');
     resolvePendingTriggerSpy.mockClear();
 
     // clear data before this run
-    const popupPage = await waitForPopup(page);
+    const popupPage = await popupPagePromise;
     await popupPage.mainFrame.waitForLoad({ loadStatus: LocationStatus.PaintingStable });
 
     // can sometimes call for paint event
@@ -668,6 +672,9 @@ describe.each([
   ['with Mitm', true],
   ['withoutMitm', false],
 ])('PaintingStable tests %s', (key, enableMitm) => {
+  if (!enableMitm) {
+    jest.retryTimes(3);
+  }
   it('should trigger painting stable after a redirect', async () => {
     const startingUrl = `${koaServer.baseUrl}/stable-redirect`;
     koaServer.get('/stable-redirect', async ctx => {

browser-emulator-builder/lib/domMatch.ts

@@ -35,7 +35,7 @@ export function isAllowedValueDifference(diff: IDiff): boolean {
   return false;
 }
 
-const explicitExportsToIgnore = ['window.navigator', 'window.chrome'];
+const explicitExportsToIgnore = ['window.chrome'];
 
 export function isExplicitExport(path: string): boolean {
   return explicitExportsToIgnore.some(x => path.startsWith(x));

browser-emulator-builder/lib/generatePolyfill.ts

@@ -172,18 +172,32 @@ function get<T>(obj: T, path: string): T {
   while (split.length) {
     const key = split.shift();
     const next = current[key];
-    if (next) current = next;
-    else if (split.length && split[0] === 'prototype') {
+
+    if (next) {
+      current = next;
+    } else if (key.startsWith('_$otherInvocation')) {
+      return current[`${key}.${split.join('.')}`];
+    } else if (split.length && split[0] === 'prototype') {
       current = current[`${key}.prototype`];
     }
   }
   return current;
 }
 
 function extractPathParts(path: string): [string, string[]] {
-  let pathParts: string[];
+  let pathParts: string[] = [];
   let propertyName: string;
-  if (path.includes('Symbol(')) {
+  if (path.includes('._$otherInvocation')) {
+    const parts = path.split('.');
+    for (const item of parts) {
+      if (item.startsWith('_$otherInvocation')) {
+        const idx = parts.indexOf(item);
+        propertyName = parts.slice(idx).join('.');
+        break;
+      }
+      pathParts.push(item);
+    }
+  } else if (path.includes('Symbol(')) {
     const symbolSplit = path.split('.Symbol(');
     propertyName = symbolSplit.pop().replace(')', '');
     pathParts = symbolSplit.shift().split('.');

browser-emulator-builder/lib/json-creators/DomPolyfill.ts

@@ -44,6 +44,7 @@ export default class DomPolyfillJson {
         DomBridger.removeHeadlessFromPolyfill(polyfill);
         // remove variations
         DomBridger.removeVariationsFromPolyfill(polyfill);
+        DomBridger.removeUnsupportedPropertiesFromPolyfill(polyfill);
 
         this.dataMap[emulateOsId] = this.dataMap[emulateOsId] || {};
         this.dataMap[emulateOsId][runtimeOsId] = polyfill;

browser-emulator-builder/package.json

@@ -4,7 +4,7 @@
   "private": true,
   "license": "MIT",
   "scripts": {
-    "generate": "node ../build/browser-emulator-builder/scripts/generateEmulatorData"
+    "generate": "node --inspect ../build/browser-emulator-builder/scripts/generateEmulatorData"
   },
   "dependencies": {
     "@double-agent/analyze": "2.0.0-alpha.28",

browser-profiler/README.md

@@ -44,10 +44,10 @@ NOTE: The latter half of this approach is still in flux and needs work to improv
 
 5. Create Bridges. Examine output in `browser-profiler-data/dom-bridges/path-patterns` and `browser-profiler-data/dom-bridges/raw-mappings` to flag anything that should be ignored when generating emulators and running double agent.
 
-- `$ yarn bridge`
+- `$ yarn workspace @ulixee/unblocked-browser-profiler-dom-bridger generate`
 
-6. Create Emulator Data (from ../browser-emulator-builder)
+1. Create Emulator Data (from ../browser-emulator-builder)
 
-- `$ yarn generate`
+- `$ yarn workspace @ulixee/unblocked-browser-emulator-builder generate`
 
 7. Commit data to Browser Profiler, Emulator Data, etc

browser-profiler/dom-bridger/index.ts

@@ -46,6 +46,30 @@ export default class DomBridger {
     polyfill.remove = polyfill.remove.filter(x => !isVariationChange(x.path, x.propertyName));
     polyfill.add = polyfill.add.filter(x => !isVariationChange(x.path, x.propertyName));
   }
+
+  public static removeUnsupportedPropertiesFromPolyfill(polyfill: IDomPolyfill): void {
+    polyfill.modify = polyfill.modify.filter(
+      x => !isUnsupportedProperty(x.path, x.propertyName, x.property),
+    );
+    polyfill.remove = polyfill.remove.filter(
+      x => !isUnsupportedProperty(x.path, x.propertyName, null),
+    );
+    polyfill.add = polyfill.add.filter(
+      x => !isUnsupportedProperty(x.path, x.propertyName, x.property),
+    );
+  }
+}
+
+function isUnsupportedProperty(path: string, propertyName: string, property: any): boolean {
+  if (property === 'Promise-like') {
+    return true;
+  }
+
+  if (typeof property === 'string' && property.includes('but only 0 present')) {
+    return true;
+  }
+
+  return false;
 }
 
 function isVariationChange(path: string, propertyName: string): boolean {
@@ -72,7 +96,8 @@ function isDevtoolsIndicator(path: string, propertyName: string): boolean {
   if (devtoolsIndicators.added.some(pattern => pathIsPatternMatch(path, pattern))) return true;
   if (devtoolsIndicators.extraAdded.some(pattern => pathIsPatternMatch(path, pattern))) return true;
   if (devtoolsIndicators.changed.some(pattern => pathIsPatternMatch(path, pattern))) return true;
-  if (devtoolsIndicators.extraChanged.some(pattern => pathIsPatternMatch(path, pattern))) return true;
+  if (devtoolsIndicators.extraChanged.some(pattern => pathIsPatternMatch(path, pattern)))
+    return true;
   return false;
 }
 

browser-profiler/dom-bridger/lib/BridgeUtils.ts

@@ -1,7 +1,7 @@
 import * as Fs from 'fs';
 import IBridgeType from '../interfaces/IBridgeType';
 
-const FOLDER_MATCH = /chrome-(8|9|10|11)[0-9]/;
+const FOLDER_MATCH = /chrome-(8|9|[1-9][0-9])[0-9]/;
 
 export function extractDirGroupsMap(
   bridge: [IBridgeType, IBridgeType],
@@ -34,7 +34,10 @@ export function extractDirGroupsMap(
 
 export function pathIsPatternMatch(path: string, pattern: string): boolean {
   if (pattern.charAt(0) === '*') {
-    return path.includes(pattern.substr(1));
+    return path.includes(pattern.slice(1));
   }
-  return path.startsWith(pattern);
+  // Split twice so we also match otherInvocationAsync, we always use otherInvocation prefix to
+  // split or match, so in case we need to encode more data (eg async) we can add it as a suffix.
+  const nestedPath = path.split('_$otherInvocation').at(1)?.split('.').slice(1).join('.');
+  return path.startsWith(pattern) || nestedPath?.startsWith(pattern);
 }

browser-profiler/dom-bridger/lib/extractors/BaseExtractor.ts

@@ -12,22 +12,17 @@ export default abstract class BaseExtractor {
 
   public patternsHandledElsewhere: string[];
 
-  public definitePathsFound: Set<string> = new Set();
-  public extraPathsFound: Set<string> = new Set();
-
-  private definitePathsMap: {
-    added: Set<string>;
-    removed: Set<string>;
-    changed: Set<string>;
-    changedOrder: Set<string>;
-  } = {
-    added: new Set(),
-    removed: new Set(),
-    changed: new Set(),
-    changedOrder: new Set(),
+  public definitePathsFound = new Set<string>();
+  public extraPathsFound = new Set<string>();
+
+  private definitePathsMap = {
+    added: new Set<string>(),
+    removed: new Set<string>(),
+    changed: new Set<string>(),
+    changedOrder: new Set<string>(),
   };
 
-  private regexpsUsedForMatch: Set<RegExp> = new Set();
+  private regexpsUsedForMatch = new Set<RegExp>();
 
   constructor(rawMappings: any) {
     for (const pathType of Object.keys(this.definitePathsMap)) {

browser-profiler/dom-bridger/lib/extractors/InstanceChangeExtractor.ts

@@ -27,6 +27,7 @@ export default class InstanceChangeExtractor extends BaseExtractor {
     'window.performance.timing.domComplete',
     'window.performance.timing.loadEventStart',
     'window.performance.timing.loadEventEnd',
+    'window.Performance.prototype.getEntries',
     'window.chrome.loadTimes',
     'window.chrome.csi',
     'window.Animation.new().ready.timeline.currentTime',
@@ -42,12 +43,21 @@ export default class InstanceChangeExtractor extends BaseExtractor {
     'window.AudioContext.new().destination.context.sampleRate',
     'window.AudioContext.new().sampleRate',
     'window.navigator.connection.rtt',
-    'window.navigation.currentEntry'
+    'window.navigation.currentEntry',
+    'window.performance.timing.toJSON',
+    'window.performance.toJSON',
+    'window.performance.now',
+    'window.document.documentElement.getInnerHTML',
+    'window.webkitRTCPeerConnection.new().createOffer',
+    'window.RTCPeerConnection.new().createOffer',
+    'window.crypto.randomUUID',
+    'window.navigator.storage.estimate',
   ];
 
   public static override extraAddPatterns = [];
 
   public static override extraChangePatterns = [
+    'window.Intl.DateTimeFormat.new().resolvedOptions',
     'window.console.memory.usedJSHeapSize',
     'window.BaseAudioContext.prototype.state',
     'window.BaseAudioContext.prototype.onstatechange',
@@ -72,6 +82,7 @@ export default class InstanceChangeExtractor extends BaseExtractor {
     'window.AudioContext.new().destination.context.currentTime',
     'window.AudioContext.new().currentTime',
     'window.document.fonts.ready',
+    'window.ScrollTimeline',
   ];
 
   public static override ignoredExtraPatterns = [];
@@ -106,9 +117,11 @@ export default class InstanceChangeExtractor extends BaseExtractor {
     /window.chrome.csi/,
     /window.chrome.loadTimes/,
 
+    /window.Intl.DateTimeFormat/,
     /navigator.appVersion/,
     /navigator.userAgent/,
     /Document.new.+lastModified/,
+    /window.ScrollTimeline/,
 
     /window.PublicKeyCredential.isUserVerifyingPlatformAuthenticatorAvailable/,
 

browser-profiler/dom-bridger/package.json

@@ -5,8 +5,8 @@
   "private": true,
   "license": "MIT",
   "scripts": {
-    "generateRawMappings": "node ../../build/browser-profiler/dom-bridger/scripts/generateRawMappings.js",
-    "generatePathPatterns": "node ../../build/browser-profiler/dom-bridger/scripts/generatePathPatterns.js",
+    "generateRawMappings": "node --inspect ../../build/browser-profiler/dom-bridger/scripts/generateRawMappings.js",
+    "generatePathPatterns": "node --inspect ../../build/browser-profiler/dom-bridger/scripts/generatePathPatterns.js",
     "generate": "yarn generateRawMappings && yarn generatePathPatterns"
   },
   "dependencies": {

browser-profiler/main/env.ts

@@ -1,10 +1,11 @@
 import '@ulixee/commons/lib/SourceMapSupport';
-import { loadEnv } from '@ulixee/commons/lib/envUtils';
+import { loadEnv, parseEnvBool } from '@ulixee/commons/lib/envUtils';
 import * as Path from 'path';
 
 loadEnv(Path.resolve(__dirname, '..'));
 const { env } = process;
 export default {
   browserStackUser: env.BROWSERSTACK_USER,
   browserStackKey: env.BROWSERSTACK_ACCESS_KEY,
+  browserStackLocal: parseEnvBool(env.BROWSERSTACK_LOCAL),
 };