Merge pull request #43890 from margelo/e2e/different-units-per-test

[NoQA] e2e: support for different units
Expensify · Jun 28, 2024 · 1d607ac · 1d607ac
2 parents 78a8208 + c97e8b5
commit 1d607ac
Show file tree

Hide file tree

Showing 15 changed files with 73 additions and 78 deletions.
diff --git a/src/libs/E2E/reactNativeLaunchingTest.ts b/src/libs/E2E/reactNativeLaunchingTest.ts
@@ -66,7 +66,7 @@ E2EClient.getTestConfig()
                 branch: Config.E2E_BRANCH,
                 name: config.name,
                 error: `Test '${config.name}' not found`,
-                isCritical: false, 
+                isCritical: false,
             });
         }
 

diff --git a/src/libs/E2E/tests/appStartTimeTest.e2e.ts b/src/libs/E2E/tests/appStartTimeTest.e2e.ts
@@ -26,7 +26,8 @@ const test = () => {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
                     name: `App start ${metric.name}`,
-                    duration: metric.duration,
+                    metric: metric.duration,
+                    unit: 'ms',
                 }),
             ),
         )

diff --git a/src/libs/E2E/tests/chatOpeningTest.e2e.ts b/src/libs/E2E/tests/chatOpeningTest.e2e.ts
@@ -49,7 +49,8 @@ const test = (config: NativeConfig) => {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
                     name: 'Chat opening',
-                    duration: entry.duration,
+                    metric: entry.duration,
+                    unit: 'ms',
                 })
                     .then(() => {
                         console.debug('[E2E] Done with chat opening, exiting…');
@@ -64,7 +65,8 @@ const test = (config: NativeConfig) => {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
                     name: 'Chat TTI',
-                    duration: entry.duration,
+                    metric: entry.duration,
+                    unit: 'ms',
                 })
                     .then(() => {
                         console.debug('[E2E] Done with chat TTI tracking, exiting…');

diff --git a/src/libs/E2E/tests/linkingTest.e2e.ts b/src/libs/E2E/tests/linkingTest.e2e.ts
@@ -75,7 +75,8 @@ const test = (config: NativeConfig) => {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
                     name: 'Comment linking',
-                    duration: entry.duration,
+                    metric: entry.duration,
+                    unit: 'ms',
                 });
 
                 switchReportResolve();

diff --git a/src/libs/E2E/tests/openChatFinderPageTest.e2e.ts b/src/libs/E2E/tests/openChatFinderPageTest.e2e.ts
@@ -44,7 +44,8 @@ const test = () => {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
                     name: 'Open Chat Finder Page TTI',
-                    duration: entry.duration,
+                    metric: entry.duration,
+                    unit: 'ms',
                 })
                     .then(() => {
                         openSearchPageResolve();
@@ -59,7 +60,8 @@ const test = () => {
                 E2EClient.submitTestResults({
                     branch: Config.E2E_BRANCH,
                     name: 'Load Search Options',
-                    duration: entry.duration,
+                    metric: entry.duration,
+                    unit: 'ms',
                 })
                     .then(() => {
                         loadSearchOptionsResolve();

diff --git a/src/libs/E2E/tests/reportTypingTest.e2e.ts b/src/libs/E2E/tests/reportTypingTest.e2e.ts
@@ -53,7 +53,8 @@ const test = (config: NativeConfig) => {
                             E2EClient.submitTestResults({
                                 branch: Config.E2E_BRANCH,
                                 name: 'Composer typing rerender count',
-                                renderCount: rerenderCount,
+                                metric: rerenderCount,
+                                unit: 'renders',
                             }).then(E2EClient.submitTestDone);
                         }, 3000);
                     })

diff --git a/src/libs/E2E/types.ts b/src/libs/E2E/types.ts
@@ -33,15 +33,17 @@ type TestModule = {default: Test};
 
 type Tests = Record<ValueOf<typeof E2EConfig.TEST_NAMES>, Test>;
 
+type Unit = 'ms' | 'MB' | '%' | 'renders' | 'FPS';
+
 type TestResult = {
     /** Name of the test */
     name: string;
 
     /** The branch where test were running */
     branch?: string;
 
-    /** Duration in milliseconds */
-    duration?: number;
+    /** The numeric value of the measurement */
+    metric?: number;
 
     /** Optional, if set indicates that the test run failed and has no valid results. */
     error?: string;
@@ -52,8 +54,8 @@ type TestResult = {
      */
     isCritical?: boolean;
 
-    /** Render count */
-    renderCount?: number;
+    /** The unit of the measurement */
+    unit?: Unit;
 };
 
-export type {SigninParams, IsE2ETestSession, NetworkCacheMap, NetworkCacheEntry, TestConfig, TestResult, TestModule, Tests};
+export type {SigninParams, IsE2ETestSession, NetworkCacheMap, NetworkCacheEntry, TestConfig, TestResult, TestModule, Tests, Unit};
diff --git a/tests/e2e/ADDING_TESTS.md b/tests/e2e/ADDING_TESTS.md
@@ -72,7 +72,8 @@ const test = () => {
       // ... do something with the measurements
       E2EClient.submitTestResults({
           name: "Navigate to report",
-          duration: measurement.duration,
+          metric: measurement.duration,
+          unit: 'ms',
       }).then(E2EClient.submitTestDone)
   });
 

diff --git a/tests/e2e/README.md b/tests/e2e/README.md
@@ -148,7 +148,8 @@ const someDurationWeCollected = // ...
 
 E2EClient.submitTestResults({
     name: 'My test name',
-    duration: someDurationWeCollected,
+    metric: someDurationWeCollected,
+    unit: 'ms',
 });
 ```
 

diff --git a/tests/e2e/compare/compare.ts b/tests/e2e/compare/compare.ts
@@ -1,3 +1,4 @@
+import type {Unit} from '@libs/E2E/types';
 import type {Stats} from '../measure/math';
 import getStats from '../measure/math';
 import * as math from './math';
@@ -28,7 +29,7 @@ const PROBABILITY_CONSIDERED_SIGNIFICANCE = 0.02;
  */
 const DURATION_DIFF_THRESHOLD_SIGNIFICANCE = 100;
 
-function buildCompareEntry(name: string, compare: Stats, baseline: Stats): Entry {
+function buildCompareEntry(name: string, compare: Stats, baseline: Stats, unit: Unit): Entry {
     const diff = compare.mean - baseline.mean;
     const relativeDurationDiff = diff / baseline.mean;
 
@@ -38,6 +39,7 @@ function buildCompareEntry(name: string, compare: Stats, baseline: Stats): Entry
     const isDurationDiffOfSignificance = prob < PROBABILITY_CONSIDERED_SIGNIFICANCE && Math.abs(diff) >= DURATION_DIFF_THRESHOLD_SIGNIFICANCE;
 
     return {
+        unit,
         name,
         baseline,
         current: compare,
@@ -50,7 +52,7 @@ function buildCompareEntry(name: string, compare: Stats, baseline: Stats): Entry
 /**
  * Compare results between baseline and current entries and categorize.
  */
-function compareResults(baselineEntries: Metric | string, compareEntries: Metric | string = baselineEntries) {
+function compareResults(baselineEntries: Metric | string, compareEntries: Metric | string = baselineEntries, metricForTest: Record<string, Unit> = {}) {
     // Unique test scenario names
     const baselineKeys = Object.keys(baselineEntries ?? {});
     const names = Array.from(new Set([...baselineKeys]));
@@ -66,7 +68,7 @@ function compareResults(baselineEntries: Metric | string, compareEntries: Metric
             const deltaStats = getStats(current);
 
             if (baseline && current) {
-                compared.push(buildCompareEntry(name, deltaStats, currentStats));
+                compared.push(buildCompareEntry(name, deltaStats, currentStats, metricForTest[name]));
             }
         });
     }
@@ -80,9 +82,9 @@ function compareResults(baselineEntries: Metric | string, compareEntries: Metric
     };
 }
 
-export default (main: Metric | string, delta: Metric | string, outputFile: string, outputFormat = 'all') => {
+export default (main: Metric | string, delta: Metric | string, outputFile: string, outputFormat = 'all', metricForTest = {}) => {
     // IMPORTANT NOTE: make sure you are passing the main/baseline results first, then the delta/compare results:
-    const outputData = compareResults(main, delta);
+    const outputData = compareResults(main, delta, metricForTest);
 
     if (outputFormat === 'console' || outputFormat === 'all') {
         printToConsole(outputData);

diff --git a/tests/e2e/compare/output/console.ts b/tests/e2e/compare/output/console.ts
@@ -1,3 +1,4 @@
+import type {Unit} from '@libs/E2E/types';
 import type {Stats} from '../../measure/math';
 import * as format from './format';
 
@@ -8,6 +9,7 @@ type Entry = {
     diff: number;
     relativeDurationDiff: number;
     isDurationDiffOfSignificance: boolean;
+    unit: Unit;
 };
 
 type Data = {
@@ -18,7 +20,7 @@ type Data = {
 };
 
 const printRegularLine = (entry: Entry) => {
-    console.debug(` - ${entry.name}: ${format.formatDurationDiffChange(entry)}`);
+    console.debug(` - ${entry.name}: ${format.formatMetricDiffChange(entry)}`);
 };
 
 /**

diff --git a/tests/e2e/compare/output/format.ts b/tests/e2e/compare/output/format.ts
@@ -20,16 +20,16 @@ const formatPercentChange = (value: number): string => {
     return `${value >= 0 ? '+' : '-'}${formatPercent(absValue)}`;
 };
 
-const formatDuration = (duration: number): string => `${duration.toFixed(3)} ms`;
+const formatMetric = (duration: number, unit: string): string => `${duration.toFixed(3)} ${unit}`;
 
-const formatDurationChange = (value: number): string => {
+const formatMetricChange = (value: number, unit: string): string => {
     if (value > 0) {
-        return `+${formatDuration(value)}`;
+        return `+${formatMetric(value, unit)}`;
     }
     if (value < 0) {
-        return `${formatDuration(value)}`;
+        return `${formatMetric(value, unit)}`;
     }
-    return '0 ms';
+    return `0 ${unit}`;
 };
 
 const formatChange = (value: number): string => {
@@ -69,18 +69,18 @@ const getDurationSymbols = (entry: Entry): string => {
     return '';
 };
 
-const formatDurationDiffChange = (entry: Entry): string => {
+const formatMetricDiffChange = (entry: Entry): string => {
     const {baseline, current} = entry;
 
-    let output = `${formatDuration(baseline.mean)} → ${formatDuration(current.mean)}`;
+    let output = `${formatMetric(baseline.mean, entry.unit)} → ${formatMetric(current.mean, entry.unit)}`;
 
     if (baseline.mean !== current.mean) {
-        output += ` (${formatDurationChange(entry.diff)}, ${formatPercentChange(entry.relativeDurationDiff)})`;
+        output += ` (${formatMetricChange(entry.diff, entry.unit)}, ${formatPercentChange(entry.relativeDurationDiff)})`;
     }
 
     output += ` ${getDurationSymbols(entry)}`;
 
     return output;
 };
 
-export {formatPercent, formatPercentChange, formatDuration, formatDurationChange, formatChange, getDurationSymbols, formatDurationDiffChange};
+export {formatPercent, formatPercentChange, formatMetric, formatMetricChange, formatChange, getDurationSymbols, formatMetricDiffChange};
diff --git a/tests/e2e/compare/output/markdown.ts b/tests/e2e/compare/output/markdown.ts
@@ -11,37 +11,37 @@ const tableHeader = ['Name', 'Duration'];
 
 const collapsibleSection = (title: string, content: string) => `<details>\n<summary>${title}</summary>\n\n${content}\n</details>\n\n`;
 
-const buildDurationDetails = (title: string, entry: Stats) => {
+const buildDurationDetails = (title: string, entry: Stats, unit: string) => {
     const relativeStdev = entry.stdev / entry.mean;
 
     return [
         `**${title}**`,
-        `Mean: ${format.formatDuration(entry.mean)}`,
-        `Stdev: ${format.formatDuration(entry.stdev)} (${format.formatPercent(relativeStdev)})`,
+        `Mean: ${format.formatMetric(entry.mean, unit)}`,
+        `Stdev: ${format.formatMetric(entry.stdev, unit)} (${format.formatPercent(relativeStdev)})`,
         entry.entries ? `Runs: ${entry.entries.join(' ')}` : '',
     ]
         .filter(Boolean)
         .join('<br/>');
 };
 
 const buildDurationDetailsEntry = (entry: Entry) =>
-    ['baseline' in entry ? buildDurationDetails('Baseline', entry.baseline) : '', 'current' in entry ? buildDurationDetails('Current', entry.current) : '']
+    ['baseline' in entry ? buildDurationDetails('Baseline', entry.baseline, entry.unit) : '', 'current' in entry ? buildDurationDetails('Current', entry.current, entry.unit) : '']
         .filter(Boolean)
         .join('<br/><br/>');
 
 const formatEntryDuration = (entry: Entry): string => {
     let formattedDuration = '';
 
     if ('baseline' in entry && 'current' in entry) {
-        formattedDuration = format.formatDurationDiffChange(entry);
+        formattedDuration = format.formatMetricDiffChange(entry);
     }
 
     if ('baseline' in entry) {
-        formattedDuration = format.formatDuration(entry.baseline.mean);
+        formattedDuration = format.formatMetric(entry.baseline.mean, entry.unit);
     }
 
     if ('current' in entry) {
-        formattedDuration = format.formatDuration(entry.current.mean);
+        formattedDuration = format.formatMetric(entry.current.mean, entry.unit);
     }
 
     return formattedDuration;

diff --git a/tests/e2e/merge.ts b/tests/e2e/merge.ts