From 93560f409ea5f590d0891a6ee6937f0933becf16 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Sat, 9 Nov 2019 16:51:22 -0800 Subject: [PATCH] core: use the same scoring curve for desktop in all channels (#9911) --- .../audits/metrics/first-contentful-paint.js | 27 ++++++++++++------- .../audits/metrics/first-cpu-idle.js | 27 ++++++++++++------- .../audits/metrics/first-meaningful-paint.js | 27 ++++++++++++------- lighthouse-core/audits/metrics/interactive.js | 27 ++++++++++++------- lighthouse-core/audits/metrics/speed-index.js | 27 ++++++++++++------- lighthouse-core/config/lr-desktop-config.js | 13 --------- .../metrics/first-contentful-paint-test.js | 12 ++++++++- .../metrics/first-meaningful-paint-test.js | 12 ++++++++- .../test/computed/metrics/interactive-test.js | 12 ++++++++- .../test/computed/metrics/speed-index-test.js | 12 ++++++++- 10 files changed, 134 insertions(+), 62 deletions(-) diff --git a/lighthouse-core/audits/metrics/first-contentful-paint.js b/lighthouse-core/audits/metrics/first-contentful-paint.js index 7cf769571658..75653041e74c 100644 --- a/lighthouse-core/audits/metrics/first-contentful-paint.js +++ b/lighthouse-core/audits/metrics/first-contentful-paint.js @@ -27,20 +27,27 @@ class FirstContentfulPaint extends Audit { title: str_(i18n.UIStrings.firstContentfulPaintMetric), description: str_(UIStrings.description), scoreDisplayMode: Audit.SCORING_MODES.NUMERIC, - requiredArtifacts: ['traces', 'devtoolsLogs'], + requiredArtifacts: ['traces', 'devtoolsLogs', 'TestedAsMobileDevice'], }; } /** - * @return {LH.Audit.ScoreOptions} + * @return {{mobile: LH.Audit.ScoreOptions, desktop: LH.Audit.ScoreOptions}} */ static get defaultOptions() { return { - // 75th and 95th percentiles HTTPArchive -> median and PODR - // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 - // see https://www.desmos.com/calculator/2t1ugwykrl - scorePODR: 2000, - scoreMedian: 4000, + mobile: { + // 75th and 95th percentiles HTTPArchive -> median and PODR + // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 + // see https://www.desmos.com/calculator/2t1ugwykrl + scorePODR: 2000, + scoreMedian: 4000, + }, + desktop: { + // SELECT QUANTILES(renderStart, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 + scorePODR: 800, + scoreMedian: 1600, + }, }; } @@ -54,12 +61,14 @@ class FirstContentfulPaint extends Audit { const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS]; const metricComputationData = {trace, devtoolsLog, settings: context.settings}; const metricResult = await ComputedFcp.request(metricComputationData, context); + const scoreOptions = + context.options[artifacts.TestedAsMobileDevice === false ? 'desktop' : 'mobile']; return { score: Audit.computeLogNormalScore( metricResult.timing, - context.options.scorePODR, - context.options.scoreMedian + scoreOptions.scorePODR, + scoreOptions.scoreMedian ), numericValue: metricResult.timing, displayValue: str_(i18n.UIStrings.seconds, {timeInMs: metricResult.timing}), diff --git a/lighthouse-core/audits/metrics/first-cpu-idle.js b/lighthouse-core/audits/metrics/first-cpu-idle.js index bab59025963f..d2a2c9582798 100644 --- a/lighthouse-core/audits/metrics/first-cpu-idle.js +++ b/lighthouse-core/audits/metrics/first-cpu-idle.js @@ -27,20 +27,27 @@ class FirstCPUIdle extends Audit { title: str_(i18n.UIStrings.firstCPUIdleMetric), description: str_(UIStrings.description), scoreDisplayMode: Audit.SCORING_MODES.NUMERIC, - requiredArtifacts: ['traces', 'devtoolsLogs'], + requiredArtifacts: ['traces', 'devtoolsLogs', 'TestedAsMobileDevice'], }; } /** - * @return {LH.Audit.ScoreOptions} + * @return {{mobile: LH.Audit.ScoreOptions, desktop: LH.Audit.ScoreOptions}} */ static get defaultOptions() { return { - // 75th and 95th percentiles HTTPArchive -> median and PODR - // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 - // see https://www.desmos.com/calculator/yv89gz2nwf - scorePODR: 2900, - scoreMedian: 6500, + mobile: { + // 75th and 95th percentiles HTTPArchive -> median and PODR + // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 + // see https://www.desmos.com/calculator/yv89gz2nwf + scorePODR: 2900, + scoreMedian: 6500, + }, + desktop: { + // SELECT QUANTILES(fullyLoaded, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 + scorePODR: 2000, + scoreMedian: 4500, + }, }; } @@ -57,12 +64,14 @@ class FirstCPUIdle extends Audit { const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS]; const metricComputationData = {trace, devtoolsLog, settings: context.settings}; const metricResult = await ComputedFci.request(metricComputationData, context); + const scoreOptions = + context.options[artifacts.TestedAsMobileDevice === false ? 'desktop' : 'mobile']; return { score: Audit.computeLogNormalScore( metricResult.timing, - context.options.scorePODR, - context.options.scoreMedian + scoreOptions.scorePODR, + scoreOptions.scoreMedian ), numericValue: metricResult.timing, displayValue: str_(i18n.UIStrings.seconds, {timeInMs: metricResult.timing}), diff --git a/lighthouse-core/audits/metrics/first-meaningful-paint.js b/lighthouse-core/audits/metrics/first-meaningful-paint.js index 2189e721b45d..0bfacc2a1fe9 100644 --- a/lighthouse-core/audits/metrics/first-meaningful-paint.js +++ b/lighthouse-core/audits/metrics/first-meaningful-paint.js @@ -27,20 +27,27 @@ class FirstMeaningfulPaint extends Audit { title: str_(i18n.UIStrings.firstMeaningfulPaintMetric), description: str_(UIStrings.description), scoreDisplayMode: Audit.SCORING_MODES.NUMERIC, - requiredArtifacts: ['traces', 'devtoolsLogs'], + requiredArtifacts: ['traces', 'devtoolsLogs', 'TestedAsMobileDevice'], }; } /** - * @return {LH.Audit.ScoreOptions} + * @return {{mobile: LH.Audit.ScoreOptions, desktop: LH.Audit.ScoreOptions}} */ static get defaultOptions() { return { - // 75th and 95th percentiles HTTPArchive -> median and PODR - // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 - // see https://www.desmos.com/calculator/2t1ugwykrl - scorePODR: 2000, - scoreMedian: 4000, + mobile: { + // 75th and 95th percentiles HTTPArchive -> median and PODR + // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 + // see https://www.desmos.com/calculator/2t1ugwykrl + scorePODR: 2000, + scoreMedian: 4000, + }, + desktop: { + // SELECT QUANTILES(renderStart, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 + scorePODR: 800, + scoreMedian: 1600, + }, }; } @@ -57,12 +64,14 @@ class FirstMeaningfulPaint extends Audit { const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS]; const metricComputationData = {trace, devtoolsLog, settings: context.settings}; const metricResult = await ComputedFmp.request(metricComputationData, context); + const scoreOptions = + context.options[artifacts.TestedAsMobileDevice === false ? 'desktop' : 'mobile']; return { score: Audit.computeLogNormalScore( metricResult.timing, - context.options.scorePODR, - context.options.scoreMedian + scoreOptions.scorePODR, + scoreOptions.scoreMedian ), numericValue: metricResult.timing, displayValue: str_(i18n.UIStrings.seconds, {timeInMs: metricResult.timing}), diff --git a/lighthouse-core/audits/metrics/interactive.js b/lighthouse-core/audits/metrics/interactive.js index 41358fd0d576..c4a1622577f2 100644 --- a/lighthouse-core/audits/metrics/interactive.js +++ b/lighthouse-core/audits/metrics/interactive.js @@ -33,20 +33,27 @@ class InteractiveMetric extends Audit { title: str_(i18n.UIStrings.interactiveMetric), description: str_(UIStrings.description), scoreDisplayMode: Audit.SCORING_MODES.NUMERIC, - requiredArtifacts: ['traces', 'devtoolsLogs'], + requiredArtifacts: ['traces', 'devtoolsLogs', 'TestedAsMobileDevice'], }; } /** - * @return {LH.Audit.ScoreOptions} + * @return {{mobile: LH.Audit.ScoreOptions, desktop: LH.Audit.ScoreOptions}} */ static get defaultOptions() { return { - // 75th and 95th percentiles HTTPArchive -> median and PODR - // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 - // see https://www.desmos.com/calculator/5xgy0pyrbp - scorePODR: 2900, - scoreMedian: 7300, + mobile: { + // 75th and 95th percentiles HTTPArchive -> median and PODR + // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 + // see https://www.desmos.com/calculator/5xgy0pyrbp + scorePODR: 2900, + scoreMedian: 7300, + }, + desktop: { + // SELECT QUANTILES(fullyLoaded, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 + scorePODR: 2000, + scoreMedian: 4500, + }, }; } @@ -61,6 +68,8 @@ class InteractiveMetric extends Audit { const metricComputationData = {trace, devtoolsLog, settings: context.settings}; const metricResult = await Interactive.request(metricComputationData, context); const timeInMs = metricResult.timing; + const scoreOptions = + context.options[artifacts.TestedAsMobileDevice === false ? 'desktop' : 'mobile']; const extendedInfo = { timeInMs, timestamp: metricResult.timestamp, @@ -73,8 +82,8 @@ class InteractiveMetric extends Audit { return { score: Audit.computeLogNormalScore( timeInMs, - context.options.scorePODR, - context.options.scoreMedian + scoreOptions.scorePODR, + scoreOptions.scoreMedian ), numericValue: timeInMs, displayValue: str_(i18n.UIStrings.seconds, {timeInMs}), diff --git a/lighthouse-core/audits/metrics/speed-index.js b/lighthouse-core/audits/metrics/speed-index.js index 3f6801defe5f..9780c069ea53 100644 --- a/lighthouse-core/audits/metrics/speed-index.js +++ b/lighthouse-core/audits/metrics/speed-index.js @@ -27,20 +27,27 @@ class SpeedIndex extends Audit { title: str_(i18n.UIStrings.speedIndexMetric), description: str_(UIStrings.description), scoreDisplayMode: Audit.SCORING_MODES.NUMERIC, - requiredArtifacts: ['traces', 'devtoolsLogs'], + requiredArtifacts: ['traces', 'devtoolsLogs', 'TestedAsMobileDevice'], }; } /** - * @return {LH.Audit.ScoreOptions} + * @return {{mobile: LH.Audit.ScoreOptions, desktop: LH.Audit.ScoreOptions}} */ static get defaultOptions() { return { - // 75th and 95th percentiles HTTPArchive -> median and PODR - // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 - // see https://www.desmos.com/calculator/orvoyu9ygq - scorePODR: 2900, - scoreMedian: 5800, + mobile: { + // 75th and 95th percentiles HTTPArchive -> median and PODR + // https://bigquery.cloud.google.com/table/httparchive:lighthouse.2018_04_01_mobile?pli=1 + // see https://www.desmos.com/calculator/orvoyu9ygq + scorePODR: 2900, + scoreMedian: 5800, + }, + desktop: { + // SELECT QUANTILES(SpeedIndex, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 + scorePODR: 1100, + scoreMedian: 2300, + }, }; } @@ -56,12 +63,14 @@ class SpeedIndex extends Audit { const devtoolsLog = artifacts.devtoolsLogs[Audit.DEFAULT_PASS]; const metricComputationData = {trace, devtoolsLog, settings: context.settings}; const metricResult = await ComputedSi.request(metricComputationData, context); + const scoreOptions = + context.options[artifacts.TestedAsMobileDevice === false ? 'desktop' : 'mobile']; return { score: Audit.computeLogNormalScore( metricResult.timing, - context.options.scorePODR, - context.options.scoreMedian + scoreOptions.scorePODR, + scoreOptions.scoreMedian ), numericValue: metricResult.timing, displayValue: str_(i18n.UIStrings.seconds, {timeInMs: metricResult.timing}), diff --git a/lighthouse-core/config/lr-desktop-config.js b/lighthouse-core/config/lr-desktop-config.js index e4de4934c262..25f959dafc4b 100644 --- a/lighthouse-core/config/lr-desktop-config.js +++ b/lighthouse-core/config/lr-desktop-config.js @@ -22,19 +22,6 @@ const config = { // Skip the h2 audit so it doesn't lie to us. See https://github.com/GoogleChrome/lighthouse/issues/6539 skipAudits: ['uses-http2'], }, - audits: [ - // 75th and 95th percentiles -> median and PODR - // SELECT QUANTILES(renderStart, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 - {path: 'metrics/first-contentful-paint', options: {scorePODR: 800, scoreMedian: 1600}}, - {path: 'metrics/first-meaningful-paint', options: {scorePODR: 800, scoreMedian: 1600}}, - // 75th and 95th percentiles -> median and PODR - // SELECT QUANTILES(SpeedIndex, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 - {path: 'metrics/speed-index', options: {scorePODR: 1100, scoreMedian: 2300}}, - // 75th and 95th percentiles -> median and PODR - // SELECT QUANTILES(fullyLoaded, 21) FROM [httparchive:summary_pages.2018_12_15_desktop] LIMIT 1000 - {path: 'metrics/interactive', options: {scorePODR: 2000, scoreMedian: 4500}}, - {path: 'metrics/first-cpu-idle', options: {scorePODR: 2000, scoreMedian: 4500}}, - ], }; module.exports = config; diff --git a/lighthouse-core/test/computed/metrics/first-contentful-paint-test.js b/lighthouse-core/test/computed/metrics/first-contentful-paint-test.js index 43dd9ea19568..a2ddb5d218d0 100644 --- a/lighthouse-core/test/computed/metrics/first-contentful-paint-test.js +++ b/lighthouse-core/test/computed/metrics/first-contentful-paint-test.js @@ -30,7 +30,7 @@ describe('Metrics: FCP', () => { assert.ok(result.pessimisticGraph, 'should have created pessimistic graph'); }); - it('should compute an observed value', async () => { + it('should compute an observed value (desktop)', async () => { const settings = {throttlingMethod: 'provided'}; const context = {settings, computedCache: new Map()}; const result = await FirstContentfulPaint.request({trace, devtoolsLog, settings}, context); @@ -38,4 +38,14 @@ describe('Metrics: FCP', () => { assert.equal(Math.round(result.timing), 499); assert.equal(result.timestamp, 225414670885); }); + + it('should compute an observed value (mobile)', async () => { + const settings = {throttlingMethod: 'provided'}; + const context = {settings, computedCache: new Map()}; + const result = await FirstContentfulPaint.request( + {trace, devtoolsLog, settings, TestedAsMobileDevice: true}, context); + + assert.equal(Math.round(result.timing), 499); + assert.equal(result.timestamp, 225414670885); + }); }); diff --git a/lighthouse-core/test/computed/metrics/first-meaningful-paint-test.js b/lighthouse-core/test/computed/metrics/first-meaningful-paint-test.js index 659df99e936a..a722b5bf836b 100644 --- a/lighthouse-core/test/computed/metrics/first-meaningful-paint-test.js +++ b/lighthouse-core/test/computed/metrics/first-meaningful-paint-test.js @@ -57,7 +57,7 @@ describe('Metrics: FMP', () => { assert.ok(result.pessimisticGraph, 'should have created pessimistic graph'); }); - it('should compute an observed value', async () => { + it('should compute an observed value (desktop)', async () => { settings = {throttlingMethod: 'provided'}; const context = {computedCache: new Map()}; const result = await FirstMeaningfulPaint.request({trace, devtoolsLog, settings}, context); @@ -66,6 +66,16 @@ describe('Metrics: FMP', () => { assert.equal(result.timestamp, 225414955343); }); + it('should compute an observed value (mobile)', async () => { + settings = {throttlingMethod: 'provided'}; + const context = {computedCache: new Map()}; + const result = await FirstMeaningfulPaint.request( + {trace, devtoolsLog, settings, TestedAsMobileDevice: true}, context); + + assert.equal(Math.round(result.timing), 783); + assert.equal(result.timestamp, 225414955343); + }); + it('handles cases when there was a tracingStartedInPage after navStart', async () => { trace = lateTracingStartedTrace; addEmptyTask(); diff --git a/lighthouse-core/test/computed/metrics/interactive-test.js b/lighthouse-core/test/computed/metrics/interactive-test.js index 904923f8943f..7a37d3800e3f 100644 --- a/lighthouse-core/test/computed/metrics/interactive-test.js +++ b/lighthouse-core/test/computed/metrics/interactive-test.js @@ -44,7 +44,7 @@ describe('Metrics: TTI', () => { assert.ok(result.pessimisticGraph, 'should have created pessimistic graph'); }); - it('should compute an observed value', async () => { + it('should compute an observed value (desktop)', async () => { const settings = {throttlingMethod: 'provided'}; const context = {settings, computedCache: new Map()}; const result = await Interactive.request({trace, devtoolsLog, settings}, context); @@ -53,6 +53,16 @@ describe('Metrics: TTI', () => { assert.equal(result.timestamp, 225415754204); }); + it('should compute an observed value (mobile)', async () => { + const settings = {throttlingMethod: 'provided'}; + const context = {settings, computedCache: new Map()}; + const result = await Interactive.request( + {trace, devtoolsLog, settings, TestedAsMobileDevice: true}, context); + + assert.equal(Math.round(result.timing), 1582); + assert.equal(result.timestamp, 225415754204); + }); + describe('#findOverlappingQuietPeriods', () => { it('should return entire range when no activity is present', () => { const navigationStart = 220023532; diff --git a/lighthouse-core/test/computed/metrics/speed-index-test.js b/lighthouse-core/test/computed/metrics/speed-index-test.js index a1c8c8a55ceb..f64c4c6e45d4 100644 --- a/lighthouse-core/test/computed/metrics/speed-index-test.js +++ b/lighthouse-core/test/computed/metrics/speed-index-test.js @@ -26,7 +26,7 @@ describe('Metrics: Speed Index', () => { }).toMatchSnapshot(); }); - it('should compute an observed value', async () => { + it('should compute an observed value (desktop)', async () => { const settings = {throttlingMethod: 'provided'}; const context = {settings, computedCache: new Map()}; const result = await SpeedIndex.request({trace, devtoolsLog, settings}, context); @@ -34,4 +34,14 @@ describe('Metrics: Speed Index', () => { assert.equal(result.timing, 605); assert.equal(result.timestamp, 225414777015); }); + + it('should compute an observed value (mobile)', async () => { + const settings = {throttlingMethod: 'provided'}; + const context = {settings, computedCache: new Map()}; + const result = await SpeedIndex.request( + {trace, devtoolsLog, settings, TestedAsMobileDevice: true}, context); + + assert.equal(result.timing, 605); + assert.equal(result.timestamp, 225414777015); + }); });