diff --git a/.cirrus.yml b/.cirrus.yml index fdf306e86789..3a9f8d3ce445 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -101,6 +101,9 @@ task: - format+analyze - name: web_benchmarks_test env: + matrix: + CHROMIUM_BUILD: "768968" # Chromium 84.0.4147.0 + CHROMIUM_BUILD: "950363" # Chromium 98.0.4758.0 CHROME_NO_SANDBOX: true install_chromium_script: - ./script/install_chromium.sh diff --git a/packages/web_benchmarks/CHANGELOG.md b/packages/web_benchmarks/CHANGELOG.md index 2cb291565b31..b70a2bc1f87a 100644 --- a/packages/web_benchmarks/CHANGELOG.md +++ b/packages/web_benchmarks/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.0.7 + +* Updates BlinkTraceEvents to match with changes in Chromium v89+ + ## 0.0.6 * Update implementation of `_RecordingWidgetsBinding` to match the [new Binding API](https://github.com/flutter/flutter/blob/master/packages/flutter/lib/src/foundation/binding.dart#L96-L128) diff --git a/packages/web_benchmarks/README.md b/packages/web_benchmarks/README.md index ba6a1cb44ade..cc38fc92cea9 100644 --- a/packages/web_benchmarks/README.md +++ b/packages/web_benchmarks/README.md @@ -5,11 +5,11 @@ benchmarks in Chrome. # Writing a benchmark -An example benchmark can be found in [test/web_benchmark_test.dart][1]. +An example benchmark can be found in [testing/web_benchmark_test.dart][1]. A web benchmark is made of two parts: a client and a server. The client is code that runs in the browser together with the benchmark code. The server serves the app's code and assets. Additionally, the server communicates with the browser to extract the performance traces. -[1]: https://github.com/flutter/packages/blob/master/packages/web_benchmarks/test/web_benchmarks_test.dart +[1]: https://github.com/flutter/packages/blob/master/packages/web_benchmarks/testing/web_benchmarks_test.dart diff --git a/packages/web_benchmarks/lib/src/browser.dart b/packages/web_benchmarks/lib/src/browser.dart index abb748d5ac27..3a92cf970e42 100644 --- a/packages/web_benchmarks/lib/src/browser.dart +++ b/packages/web_benchmarks/lib/src/browser.dart @@ -554,7 +554,8 @@ class BlinkTraceEvent { bool get isBeginFrame => ph == 'X' && (name == 'WebViewImpl::beginFrame' || - name == 'WebFrameWidgetBase::BeginMainFrame'); + name == 'WebFrameWidgetBase::BeginMainFrame' || + name == 'WebFrameWidgetImpl::BeginMainFrame'); /// An "update all lifecycle phases" event contains UI thread computations /// related to an animation frame that's outside the scripting phase. @@ -564,7 +565,9 @@ class BlinkTraceEvent { /// /// This event is a duration event that has its `tdur` populated. bool get isUpdateAllLifecyclePhases => - ph == 'X' && name == 'WebViewImpl::updateAllLifecyclePhases'; + ph == 'X' && + (name == 'WebViewImpl::updateAllLifecyclePhases' || + name == 'WebFrameWidgetImpl::UpdateLifecycle'); /// Whether this is the beginning of a "measured_frame" event. /// diff --git a/packages/web_benchmarks/pubspec.yaml b/packages/web_benchmarks/pubspec.yaml index b83cbf7773a6..864607893ddb 100644 --- a/packages/web_benchmarks/pubspec.yaml +++ b/packages/web_benchmarks/pubspec.yaml @@ -2,7 +2,7 @@ name: web_benchmarks description: A benchmark harness for performance-testing Flutter apps in Chrome. repository: https://github.com/flutter/packages/tree/main/packages/web_benchmarks issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+web_benchmarks%22 -version: 0.0.6 +version: 0.0.7 environment: sdk: ">=2.7.0 <3.0.0" diff --git a/packages/web_benchmarks/test/more_tests_in_testing_dir_test.dart b/packages/web_benchmarks/test/more_tests_in_testing_dir_test.dart new file mode 100644 index 000000000000..d7c658c63b82 --- /dev/null +++ b/packages/web_benchmarks/test/more_tests_in_testing_dir_test.dart @@ -0,0 +1,14 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('Tell the user how to run tests within "testing" directory', () { + print('---'); + print('This package also has client-server tests.'); + print('Use `flutter test testing` to run them.'); + print('---'); + }); +} diff --git a/packages/web_benchmarks/test/src/browser_test.dart b/packages/web_benchmarks/test/src/browser_test.dart new file mode 100644 index 000000000000..a92399783884 --- /dev/null +++ b/packages/web_benchmarks/test/src/browser_test.dart @@ -0,0 +1,55 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:web_benchmarks/src/browser.dart'; + +import 'browser_test_json_samples.dart'; + +void main() { + group('BlinkTraceEvent works with Chrome 89+', () { + // Used to test 'false' results + final BlinkTraceEvent unrelatedPhX = + BlinkTraceEvent.fromJson(unrelatedPhXJson); + final BlinkTraceEvent anotherUnrelated = + BlinkTraceEvent.fromJson(anotherUnrelatedJson); + + test('isBeginFrame', () { + final BlinkTraceEvent event = + BlinkTraceEvent.fromJson(beginMainFrameJson_89plus); + + expect(event.isBeginFrame, isTrue); + expect(unrelatedPhX.isBeginFrame, isFalse); + expect(anotherUnrelated.isBeginFrame, isFalse); + }); + + test('isUpdateAllLifecyclePhases', () { + final BlinkTraceEvent event = + BlinkTraceEvent.fromJson(updateLifecycleJson_89plus); + + expect(event.isUpdateAllLifecyclePhases, isTrue); + expect(unrelatedPhX.isUpdateAllLifecyclePhases, isFalse); + expect(anotherUnrelated.isUpdateAllLifecyclePhases, isFalse); + }); + + test('isBeginMeasuredFrame', () { + final BlinkTraceEvent event = + BlinkTraceEvent.fromJson(beginMeasuredFrameJson_89plus); + + expect(event.isBeginMeasuredFrame, isTrue); + expect(unrelatedPhX.isBeginMeasuredFrame, isFalse); + expect(anotherUnrelated.isBeginMeasuredFrame, isFalse); + }); + + test('isEndMeasuredFrame', () { + final BlinkTraceEvent event = + BlinkTraceEvent.fromJson(endMeasuredFrameJson_89plus); + + expect(event.isEndMeasuredFrame, isTrue); + expect(unrelatedPhX.isEndMeasuredFrame, isFalse); + expect(anotherUnrelated.isEndMeasuredFrame, isFalse); + }); + }); +} diff --git a/packages/web_benchmarks/test/src/browser_test_json_samples.dart b/packages/web_benchmarks/test/src/browser_test_json_samples.dart new file mode 100644 index 000000000000..8152aa16bf51 --- /dev/null +++ b/packages/web_benchmarks/test/src/browser_test_json_samples.dart @@ -0,0 +1,103 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert' show jsonDecode; + +// JSON Event samples taken from running an instrumented version of the +// integration tests of this package that dumped all the data as captured. + +/// To test isBeginFrame. (Sampled from Chrome 89+) +final Map beginMainFrameJson_89plus = jsonDecode(''' +{ + "args": { + "frameTime": 2338687248768 + }, + "cat": "blink", + "dur": 6836, + "name": "WebFrameWidgetImpl::BeginMainFrame", + "ph": "X", + "pid": 1367081, + "tdur": 393, + "tid": 1, + "ts": 2338687258440, + "tts": 375499 +} +'''); + +/// To test isUpdateAllLifecyclePhases. (Sampled from Chrome 89+) +final Map updateLifecycleJson_89plus = jsonDecode(''' +{ + "args": {}, + "cat": "blink", + "dur": 103, + "name": "WebFrameWidgetImpl::UpdateLifecycle", + "ph": "X", + "pid": 1367081, + "tdur": 102, + "tid": 1, + "ts": 2338687265284, + "tts": 375900 +} +'''); + +/// To test isBeginMeasuredFrame. (Sampled from Chrome 89+) +final Map beginMeasuredFrameJson_89plus = jsonDecode(''' +{ + "args": {}, + "cat": "blink.user_timing", + "id": "0xea2a8b45", + "name": "measured_frame", + "ph": "b", + "pid": 1367081, + "scope": "blink.user_timing", + "tid": 1, + "ts": 2338687265932 +} +'''); + +/// To test isEndMeasuredFrame. (Sampled from Chrome 89+) +final Map endMeasuredFrameJson_89plus = jsonDecode(''' +{ + "args": {}, + "cat": "blink.user_timing", + "id": "0xea2a8b45", + "name": "measured_frame", + "ph": "e", + "pid": 1367081, + "scope": "blink.user_timing", + "tid": 1, + "ts": 2338687440485 +} +'''); + +/// An unrelated data frame to test negative cases. +final Map unrelatedPhXJson = jsonDecode(''' +{ + "args": {}, + "cat": "blink,rail", + "dur": 2, + "name": "PageAnimator::serviceScriptedAnimations", + "ph": "X", + "pid": 1367081, + "tdur": 2, + "tid": 1, + "ts": 2338691143317, + "tts": 1685405 +} +'''); + +/// Another unrelated data frame to test negative cases. +final Map anotherUnrelatedJson = jsonDecode(''' +{ + "args": { + "sort_index": -1 + }, + "cat": "__metadata", + "name": "thread_sort_index", + "ph": "M", + "pid": 1367081, + "tid": 1, + "ts": 2338692906482 +} +'''); diff --git a/packages/web_benchmarks/testing/README.md b/packages/web_benchmarks/testing/README.md new file mode 100644 index 000000000000..58e8429952c5 --- /dev/null +++ b/packages/web_benchmarks/testing/README.md @@ -0,0 +1,72 @@ +# `testing` README + +## How to run the `testing` directory tests + +The benchmarks contained in this directory use a client-server model, similar to +what the integration_test package does. In order to run the tests inside `testing`, +do the following: + +* Install Chrome in a way that [tests can find it](https://github.com/flutter/packages/blob/a5a4479e176c5e909dd5d961c2c79b61ce1bf1bd/packages/web_benchmarks/lib/src/browser.dart#L216). + +* Fetch dependencies for the `test_app` directory inside `testing`: + + ```bash + flutter pub get testing/test_app + ``` + +* Fetch dependencies for the `web_benchmarks` directory: + + ```bash + flutter pub get + ``` + +* Run the tests with `flutter test`: + + ```bash + $ flutter test testing + + 00:03 +0: Can run a web benchmark + Launching Chrome. + Launching Google Chrome 98.0.4758.102 + + Waiting for the benchmark to report benchmark profile. + [CHROME]: [0215/133233.327761:ERROR:socket_posix.cc(150)] bind() failed: Address already in use (98) + [CHROME]: + [CHROME]: DevTools listening on ws://[::1]:10000/devtools/browser/4ef82be6-9b68-4fd3-ab90-cd603d25ceb1 + Connecting to DevTools: ws://localhost:10000/devtools/page/21E7271507E9BC796B957E075515520F + Connected to Chrome tab: (http://localhost:9999/index.html) + Launching benchmark "scroll" + Extracted 299 measured frames. + Skipped 1 non-measured frames. + Launching benchmark "page" + [APP] Testing round 0... + [APP] Testing round 1... + [APP] Testing round 2... + [APP] Testing round 3... + [APP] Testing round 4... + [APP] Testing round 5... + [APP] Testing round 6... + [APP] Testing round 7... + [APP] Testing round 8... + [APP] Testing round 9... + Extracted 490 measured frames. + Skipped 0 non-measured frames. + Launching benchmark "tap" + [APP] Testing round 0... + [APP] Testing round 1... + [APP] Testing round 2... + [APP] Testing round 3... + [APP] Testing round 4... + [APP] Testing round 5... + [APP] Testing round 6... + [APP] Testing round 7... + [APP] Testing round 8... + [APP] Testing round 9... + Extracted 299 measured frames. + Skipped 0 non-measured frames. + Received profile data + 00:26 +1: All tests passed! + ``` + +_(If the above stops working, take a look at what the [`web_benchmarks_test` Cirrus step](https://github.com/flutter/packages/blob/a5a4479e176c5e909dd5d961c2c79b61ce1bf1bd/.cirrus.yml#L102-L113) +is currently doing, and update this document accordingly!)_ diff --git a/script/install_chromium.sh b/script/install_chromium.sh index 573e87c74360..710d06c63af9 100755 --- a/script/install_chromium.sh +++ b/script/install_chromium.sh @@ -8,7 +8,11 @@ set -x # The build of Chromium used to test web functionality. # # Chromium builds can be located here: https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/ -CHROMIUM_BUILD=768968 +# +# Set CHROMIUM_BUILD env-var to let the script know what to download. + +: "${CHROMIUM_BUILD:=950363}" # Default value for the CHROMIUM_BUILD env. +echo "Downloading CHROMIUM_BUILD=${CHROMIUM_BUILD}" mkdir .chromium wget "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${CHROMIUM_BUILD}%2Fchrome-linux.zip?alt=media" -O .chromium/chromium.zip