From 53e6876226a7729531be7be4fa7321363c5a3b30 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 7 Nov 2022 19:14:03 -0600 Subject: [PATCH] Allow Flutter golden file tests to be flaky (#114450) * allow marking a golden check as flaky * add matchesFlutterGolden to analyze.dart; no tags for flutter_goldens/lib * Pause * ++ * ++ * Analyzer therapy * Once more with feeling * Nits * Review feedback * Silly oops * Test progress * More tests * Finish * Nits * Analyzer * Review feedback Co-authored-by: Yegor Jbanov --- .../flutter_test/flutter_gold_test.dart | 2 +- dev/bots/analyze.dart | 15 +- .../packages/foo/flaky_golden_ignore.dart | 16 + .../foo/flaky_golden_missing_tag.dart | 14 + .../packages/foo/flaky_golden_no_tag.dart | 8 +- .../root/packages/foo/golden_class.dart | 4 + .../root/packages/foo/golden_doc.dart | 3 + dev/bots/test/analyze_test.dart | 2 + .../bin/tasks/technical_debt__cost.dart | 4 + examples/api/test/flutter_test_config.dart | 2 +- examples/api/test/goldens_web.dart | 8 - packages/flutter/test/_goldens_io.dart | 5 - packages/flutter/test/_goldens_web.dart | 8 - .../test/cupertino/date_picker_test.dart | 48 +- .../flutter/test/flutter_test_config.dart | 8 +- .../flutter/test/flutter_web_test_config.dart | 25 + .../flutter_goldens/lib/flutter_goldens.dart | 544 +----------- .../lib/src/flaky_goldens.dart | 98 +++ .../lib/src/flutter_goldens_io.dart | 611 ++++++++++++++ .../lib/src/flutter_goldens_web.dart | 71 ++ .../test/flaky_goldens_test.dart | 52 ++ .../test/flutter_goldens_test.dart | 797 ++---------------- .../test/skia_client_test.dart | 716 ++++++++++++++++ .../flutter_goldens/test/utils/fakes.dart | 205 +++++ .../test/{ => utils}/json_templates.dart | 0 .../lib/skia_client.dart | 63 +- .../flutter_test/lib/src/_goldens_web.dart | 6 +- packages/flutter_test/lib/src/goldens.dart | 8 + .../lib/src/test/flutter_web_goldens.dart | 36 +- .../lib/src/test/flutter_web_platform.dart | 12 +- .../lib/src/test/test_config.dart | 19 +- .../flutter_tools/lib/src/web/bootstrap.dart | 3 +- .../web/golden_comparator_process_test.dart | 15 +- .../web/golden_comparator_test.dart | 14 +- .../forbidden_imports_test.dart | 2 +- 35 files changed, 2093 insertions(+), 1351 deletions(-) create mode 100644 dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_ignore.dart create mode 100644 dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart rename examples/api/test/goldens_io.dart => dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart (53%) delete mode 100644 examples/api/test/goldens_web.dart delete mode 100644 packages/flutter/test/_goldens_io.dart delete mode 100644 packages/flutter/test/_goldens_web.dart create mode 100644 packages/flutter/test/flutter_web_test_config.dart create mode 100644 packages/flutter_goldens/lib/src/flaky_goldens.dart create mode 100644 packages/flutter_goldens/lib/src/flutter_goldens_io.dart create mode 100644 packages/flutter_goldens/lib/src/flutter_goldens_web.dart create mode 100644 packages/flutter_goldens/test/flaky_goldens_test.dart create mode 100644 packages/flutter_goldens/test/skia_client_test.dart create mode 100644 packages/flutter_goldens/test/utils/fakes.dart rename packages/flutter_goldens/test/{ => utils}/json_templates.dart (100%) diff --git a/dev/automated_tests/flutter_test/flutter_gold_test.dart b/dev/automated_tests/flutter_test/flutter_gold_test.dart index b12ac4efd169..04778f5a6250 100644 --- a/dev/automated_tests/flutter_test/flutter_gold_test.dart +++ b/dev/automated_tests/flutter_test/flutter_gold_test.dart @@ -8,7 +8,7 @@ import 'dart:typed_data'; import 'package:file/file.dart'; import 'package:file/memory.dart'; -import 'package:flutter_goldens/flutter_goldens.dart'; +import 'package:flutter_goldens/src/flutter_goldens_io.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:platform/platform.dart'; diff --git a/dev/bots/analyze.dart b/dev/bots/analyze.dart index f6c9ec227d04..f3dd63740a88 100644 --- a/dev/bots/analyze.dart +++ b/dev/bots/analyze.dart @@ -418,8 +418,8 @@ Future verifyNoSyncAsyncStar(String workingDirectory, {int minimumMatches } } -final RegExp _findGoldenTestPattern = RegExp(r'matchesGoldenFile\('); -final RegExp _findGoldenDefinitionPattern = RegExp(r'matchesGoldenFile\(Object'); +final RegExp _findGoldenTestPattern = RegExp(r'(matchesGoldenFile|expectFlakyGolden)\('); +final RegExp _findGoldenDefinitionPattern = RegExp(r'(matchesGoldenFile|expectFlakyGolden)\(Object'); final RegExp _leadingComment = RegExp(r'//'); final RegExp _goldenTagPattern1 = RegExp(r'@Tags\('); final RegExp _goldenTagPattern2 = RegExp(r"'reduced-test-set'"); @@ -431,8 +431,17 @@ const String _ignoreGoldenTag = '// flutter_ignore: golden_tag (see analyze.dart const String _ignoreGoldenTagForFile = '// flutter_ignore_for_file: golden_tag (see analyze.dart)'; Future verifyGoldenTags(String workingDirectory, { int minimumMatches = 2000 }) async { + // Skip flutter_goldens/lib because this library uses `matchesGoldenFile` + // but is not itself a test that needs tags. + final String flutterGoldensPackageLib = path.join(flutterPackages, 'flutter_goldens', 'lib'); + bool isWithinFlutterGoldenLib(File file) { + return path.isWithin(flutterGoldensPackageLib, file.path); + } + final List errors = []; - await for (final File file in _allFiles(workingDirectory, 'dart', minimumMatches: minimumMatches)) { + final Stream allTestFiles = _allFiles(workingDirectory, 'dart', minimumMatches: minimumMatches) + .where((File file) => !isWithinFlutterGoldenLib(file)); + await for (final File file in allTestFiles) { bool needsTag = false; bool hasTagNotation = false; bool hasReducedTag = false; diff --git a/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_ignore.dart b/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_ignore.dart new file mode 100644 index 000000000000..b1601c631de1 --- /dev/null +++ b/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_ignore.dart @@ -0,0 +1,16 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This would fail analysis, but it is ignored +// flutter_ignore_for_file: golden_tag (see analyze.dart) + +@Tags(['some-other-tag']) + +import 'package:test/test.dart'; + +import 'golden_class.dart'; + +void main() { + expectFlakyGolden('key', 'String'); +} diff --git a/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart b/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart new file mode 100644 index 000000000000..c70afc0cb5b0 --- /dev/null +++ b/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// The reduced test set tag is missing. This should fail analysis. +@Tags(['some-other-tag']) + +import 'package:test/test.dart'; + +import 'golden_class.dart'; + +void main() { + expectFlakyGolden('finder', 'missing_tag.png'); +} diff --git a/examples/api/test/goldens_io.dart b/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart similarity index 53% rename from examples/api/test/goldens_io.dart rename to dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart index d552235d6903..f55bae26d565 100644 --- a/examples/api/test/goldens_io.dart +++ b/dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart @@ -2,4 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -export 'package:flutter_goldens/flutter_goldens.dart' show testExecutable; +// The tag is missing. This should fail analysis. + +import 'golden_class.dart'; + +void main() { + expectFlakyGolden('key', 'missing_tag.png'); +} diff --git a/dev/bots/test/analyze-test-input/root/packages/foo/golden_class.dart b/dev/bots/test/analyze-test-input/root/packages/foo/golden_class.dart index 6d5b7630543a..54e105ec6104 100644 --- a/dev/bots/test/analyze-test-input/root/packages/foo/golden_class.dart +++ b/dev/bots/test/analyze-test-input/root/packages/foo/golden_class.dart @@ -7,3 +7,7 @@ void matchesGoldenFile(Object key) { return; } + +void expectFlakyGolden(Object key, String string){ + return; +} diff --git a/dev/bots/test/analyze-test-input/root/packages/foo/golden_doc.dart b/dev/bots/test/analyze-test-input/root/packages/foo/golden_doc.dart index f6a365b457e9..fbd1a39d2939 100644 --- a/dev/bots/test/analyze-test-input/root/packages/foo/golden_doc.dart +++ b/dev/bots/test/analyze-test-input/root/packages/foo/golden_doc.dart @@ -35,8 +35,11 @@ /// ``` /// {@end-tool} /// +/// expectFlakyGolden(a, b) // Other comments // matchesGoldenFile('comment.png'); +// expectFlakyGolden(a, b); String literal = 'matchesGoldenFile()'; // flutter_ignore: golden_tag (see analyze.dart) +String flakyLiteral = 'expectFlakyGolden'; diff --git a/dev/bots/test/analyze_test.dart b/dev/bots/test/analyze_test.dart index ace3c7a3fead..d62918ec91fb 100644 --- a/dev/bots/test/analyze_test.dart +++ b/dev/bots/test/analyze_test.dart @@ -79,7 +79,9 @@ void main() { 'at the top of the file before import statements.'; const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'."; final List lines = [ + '║ test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart: $noTag', '║ test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag', + '║ test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart: $missingTag', '║ test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag', ] .map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/')) diff --git a/dev/devicelab/bin/tasks/technical_debt__cost.dart b/dev/devicelab/bin/tasks/technical_debt__cost.dart index ff113b32d64c..4b2afaa725d0 100644 --- a/dev/devicelab/bin/tasks/technical_debt__cost.dart +++ b/dev/devicelab/bin/tasks/technical_debt__cost.dart @@ -17,6 +17,7 @@ const double todoCost = 1009.0; // about two average SWE days, in dollars const double ignoreCost = 2003.0; // four average SWE days, in dollars const double pythonCost = 3001.0; // six average SWE days, in dollars const double skipCost = 2473.0; // 20 hours: 5 to fix the issue we're ignoring, 15 to fix the bugs we missed because the test was off +const double flakyGoldenCost = 2467.0; // Similar to skip cost const double ignoreForFileCost = 2477.0; // similar thinking as skipCost const double asDynamicCost = 2011.0; // a few days to refactor the code. const double deprecationCost = 233.0; // a few hours to remove the old code. @@ -69,6 +70,9 @@ Future findCostsForFile(File file) async { if (isTest && line.contains('skip:') && !line.contains('[intended]')) { total += skipCost; } + if (isTest && line.contains('expectFlakyGolden(')) { + total += flakyGoldenCost; + } if (isDart && isOptingOutOfNullSafety(line)) { total += fileNullSafetyMigrationCost; } diff --git a/examples/api/test/flutter_test_config.dart b/examples/api/test/flutter_test_config.dart index 69ad13655ba3..1b7fdd6b7347 100644 --- a/examples/api/test/flutter_test_config.dart +++ b/examples/api/test/flutter_test_config.dart @@ -5,7 +5,7 @@ import 'dart:async'; -import 'goldens_io.dart' if (dart.library.html) 'goldens_web.dart' as flutter_goldens; +import 'package:flutter_goldens/flutter_goldens.dart' as flutter_goldens; Future testExecutable(FutureOr Function() testMain) { // Enable golden file testing using Skia Gold. diff --git a/examples/api/test/goldens_web.dart b/examples/api/test/goldens_web.dart deleted file mode 100644 index 57c0fbad759f..000000000000 --- a/examples/api/test/goldens_web.dart +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; - -// package:flutter_goldens is not used as part of the test process for web. -Future testExecutable(FutureOr Function() testMain) async => testMain(); diff --git a/packages/flutter/test/_goldens_io.dart b/packages/flutter/test/_goldens_io.dart deleted file mode 100644 index d552235d6903..000000000000 --- a/packages/flutter/test/_goldens_io.dart +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -export 'package:flutter_goldens/flutter_goldens.dart' show testExecutable; diff --git a/packages/flutter/test/_goldens_web.dart b/packages/flutter/test/_goldens_web.dart deleted file mode 100644 index 57c0fbad759f..000000000000 --- a/packages/flutter/test/_goldens_web.dart +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; - -// package:flutter_goldens is not used as part of the test process for web. -Future testExecutable(FutureOr Function() testMain) async => testMain(); diff --git a/packages/flutter/test/cupertino/date_picker_test.dart b/packages/flutter/test/cupertino/date_picker_test.dart index 3353f6507a0e..461248aa7799 100644 --- a/packages/flutter/test/cupertino/date_picker_test.dart +++ b/packages/flutter/test/cupertino/date_picker_test.dart @@ -17,10 +17,11 @@ import 'dart:ui'; import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; +import 'package:flutter_goldens/flutter_goldens.dart' show expectFlakyGolden; import 'package:flutter_test/flutter_test.dart'; // TODO(yjbanov): on the web text rendered with perspective produces flaky goldens: https://github.com/flutter/flutter/issues/110785 -const bool skipPerspectiveTextGoldens = isBrowser; +const bool perspectiveTestIsFlaky = isBrowser; // A number of the hit tests below say "warnIfMissed: false". This is because // the way the CupertinoPicker works, the hits don't actually reach the labels, @@ -1197,7 +1198,13 @@ void main() { } await tester.pumpWidget(buildApp(CupertinoDatePickerMode.time)); - if (!skipPerspectiveTextGoldens) { + + if (perspectiveTestIsFlaky) { + await expectFlakyGolden( + find.byType(CupertinoDatePicker), + 'date_picker_test.time.initial.png', + ); + } else { await expectLater( find.byType(CupertinoDatePicker), matchesGoldenFile('date_picker_test.time.initial.png'), @@ -1205,7 +1212,13 @@ void main() { } await tester.pumpWidget(buildApp(CupertinoDatePickerMode.date)); - if (!skipPerspectiveTextGoldens) { + + if (perspectiveTestIsFlaky) { + await expectFlakyGolden( + find.byType(CupertinoDatePicker), + 'date_picker_test.date.initial.png', + ); + } else { await expectLater( find.byType(CupertinoDatePicker), matchesGoldenFile('date_picker_test.date.initial.png'), @@ -1213,7 +1226,13 @@ void main() { } await tester.pumpWidget(buildApp(CupertinoDatePickerMode.dateAndTime)); - if (!skipPerspectiveTextGoldens) { + + if (perspectiveTestIsFlaky) { + await expectFlakyGolden( + find.byType(CupertinoDatePicker), + 'date_picker_test.datetime.initial.png', + ); + } else { await expectLater( find.byType(CupertinoDatePicker), matchesGoldenFile('date_picker_test.datetime.initial.png'), @@ -1224,7 +1243,12 @@ void main() { await tester.drag(find.text('4'), Offset(0, _kRowOffset.dy / 2), warnIfMissed: false); // see top of file await tester.pump(); - if (!skipPerspectiveTextGoldens) { + if (perspectiveTestIsFlaky) { + await expectFlakyGolden( + find.byType(CupertinoDatePicker), + 'date_picker_test.datetime.drag.png', + ); + } else { await expectLater( find.byType(CupertinoDatePicker), matchesGoldenFile('date_picker_test.datetime.drag.png'), @@ -1314,7 +1338,12 @@ void main() { ), ); - if (!skipPerspectiveTextGoldens) { + if (perspectiveTestIsFlaky) { + await expectFlakyGolden( + find.byType(CupertinoTimerPicker), + 'timer_picker_test.datetime.initial.png', + ); + } else { await expectLater( find.byType(CupertinoTimerPicker), matchesGoldenFile('timer_picker_test.datetime.initial.png'), @@ -1325,7 +1354,12 @@ void main() { await tester.drag(find.text('59'), Offset(0, _kRowOffset.dy / 2), warnIfMissed: false); // see top of file await tester.pump(); - if (!skipPerspectiveTextGoldens) { + if (perspectiveTestIsFlaky) { + await expectFlakyGolden( + find.byType(CupertinoTimerPicker), + 'timer_picker_test.datetime.drag.png', + ); + } else { await expectLater( find.byType(CupertinoTimerPicker), matchesGoldenFile('timer_picker_test.datetime.drag.png'), diff --git a/packages/flutter/test/flutter_test_config.dart b/packages/flutter/test/flutter_test_config.dart index e6c0c9855e22..bcea1ef49b67 100644 --- a/packages/flutter/test/flutter_test_config.dart +++ b/packages/flutter/test/flutter_test_config.dart @@ -5,11 +5,9 @@ import 'dart:async'; import 'package:flutter/rendering.dart'; +import 'package:flutter_goldens/flutter_goldens.dart' as flutter_goldens; import 'package:flutter_test/flutter_test.dart'; -import '_goldens_io.dart' - if (dart.library.html) '_goldens_web.dart' as flutter_goldens; - Future testExecutable(FutureOr Function() testMain) { // Enable checks because there are many implementations of [RenderBox] in this // package can benefit from the additional validations. @@ -22,3 +20,7 @@ Future testExecutable(FutureOr Function() testMain) { // Enable golden file testing using Skia Gold. return flutter_goldens.testExecutable(testMain); } + +Future processBrowserCommand(dynamic command) { + return flutter_goldens.processBrowserCommand(command); +} diff --git a/packages/flutter/test/flutter_web_test_config.dart b/packages/flutter/test/flutter_web_test_config.dart new file mode 100644 index 000000000000..b4d083073b72 --- /dev/null +++ b/packages/flutter/test/flutter_web_test_config.dart @@ -0,0 +1,25 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'flutter_test_config.dart'; + +/// A custom host configuration for browser tests that supports flaky golden +/// checks. +/// +/// See also [processBrowserCommand]. +Future startWebTestHostConfiguration(String testUri) async { + testExecutable(() async { + final Stream commands = stdin + .transform(utf8.decoder) + .transform(const LineSplitter()) + .map(jsonDecode); + await for (final dynamic command in commands) { + await processBrowserCommand(command); + } + }); +} diff --git a/packages/flutter_goldens/lib/flutter_goldens.dart b/packages/flutter_goldens/lib/flutter_goldens.dart index b57f9a1a611f..75509ee30f01 100644 --- a/packages/flutter_goldens/lib/flutter_goldens.dart +++ b/packages/flutter_goldens/lib/flutter_goldens.dart @@ -2,544 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:async' show FutureOr; -import 'dart:io' as io show OSError, SocketException; - -import 'package:file/file.dart'; -import 'package:file/local.dart'; -import 'package:flutter/foundation.dart'; -import 'package:flutter_goldens_client/skia_client.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:platform/platform.dart'; +/// This library exposes functions that enhance the test with custom golden +/// configuration for the Flutter repository. export 'package:flutter_goldens_client/skia_client.dart'; - -// If you are here trying to figure out how to use golden files in the Flutter -// repo itself, consider reading this wiki page: -// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter - -const String _kFlutterRootKey = 'FLUTTER_ROOT'; - -/// Main method that can be used in a `flutter_test_config.dart` file to set -/// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that -/// works for the current test. _Which_ FlutterGoldenFileComparator is -/// instantiated is based on the current testing environment. -/// -/// When set, the `namePrefix` is prepended to the names of all gold images. -Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async { - const Platform platform = LocalPlatform(); - if (FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform)) { - goldenFileComparator = await FlutterPostSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix); - } else if (FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform)) { - goldenFileComparator = await FlutterPreSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix); - } else if (FlutterSkippingFileComparator.isAvailableForEnvironment(platform)) { - goldenFileComparator = FlutterSkippingFileComparator.fromDefaultComparator( - 'Golden file testing is not executed on Cirrus, or LUCI environments outside of flutter/flutter.', - namePrefix: namePrefix - ); - } else { - goldenFileComparator = await FlutterLocalFileComparator.fromDefaultComparator(platform); - } - - await testMain(); -} - -/// Abstract base class golden file comparator specific to the `flutter/flutter` -/// repository. -/// -/// Golden file testing for the `flutter/flutter` repository is handled by three -/// different [FlutterGoldenFileComparator]s, depending on the current testing -/// environment. -/// -/// * The [FlutterPostSubmitFileComparator] is utilized during post-submit -/// testing, after a pull request has landed on the master branch. This -/// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload -/// tests to the [Flutter Gold dashboard](https://flutter-gold.skia.org). -/// Flutter Gold manages the master golden files for the `flutter/flutter` -/// repository. -/// -/// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing, -/// before a pull request lands on the master branch. This -/// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing -/// contributors to view and check in visual differences before landing the -/// change. -/// -/// * The [FlutterLocalFileComparator] is used for local development testing. -/// This comparator will use the [SkiaGoldClient] to request baseline images -/// from [Flutter Gold](https://flutter-gold.skia.org) and manually compare -/// pixels. If a difference is detected, this comparator will -/// generate failure output illustrating the found difference. If a baseline -/// is not found for a given test image, it will consider it a new test and -/// output the new image for verification. -/// -/// The [FlutterSkippingFileComparator] is utilized to skip tests outside -/// of the appropriate environments described above. Currently, some Luci -/// environments do not execute golden file testing, and as such do not require -/// a comparator. This comparator is also used when an internet connection is -/// unavailable. -abstract class FlutterGoldenFileComparator extends GoldenFileComparator { - /// Creates a [FlutterGoldenFileComparator] that will resolve golden file - /// URIs relative to the specified [basedir], and retrieve golden baselines - /// using the [skiaClient]. The [basedir] is used for writing and accessing - /// information and files for interacting with the [skiaClient]. When testing - /// locally, the [basedir] will also contain any diffs from failed tests, or - /// goldens generated from newly introduced tests. - /// - /// The [fs] and [platform] parameters are useful in tests, where the default - /// file system and platform can be replaced by mock instances. - @visibleForTesting - FlutterGoldenFileComparator( - this.basedir, - this.skiaClient, { - this.fs = const LocalFileSystem(), - this.platform = const LocalPlatform(), - this.namePrefix, - }); - - /// The directory to which golden file URIs will be resolved in [compare] and - /// [update], cannot be null. - final Uri basedir; - - /// A client for uploading image tests and making baseline requests to the - /// Flutter Gold Dashboard, cannot be null. - final SkiaGoldClient skiaClient; - - /// The file system used to perform file access. - @visibleForTesting - final FileSystem fs; - - /// A wrapper for the [dart:io.Platform] API. - @visibleForTesting - final Platform platform; - - /// The prefix that is added to all golden names. - final String? namePrefix; - - @override - Future update(Uri golden, Uint8List imageBytes) async { - final File goldenFile = getGoldenFile(golden); - await goldenFile.parent.create(recursive: true); - await goldenFile.writeAsBytes(imageBytes, flush: true); - } - - @override - Uri getTestUri(Uri key, int? version) => key; - - /// Calculate the appropriate basedir for the current test context. - /// - /// The optional [suffix] argument is used by the - /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator]. - /// These [FlutterGoldenFileComparators] randomize their base directories to - /// maintain thread safety while using the `goldctl` tool. - @protected - @visibleForTesting - static Directory getBaseDirectory( - LocalFileComparator defaultComparator, - Platform platform, { - String? suffix, - }) { - const FileSystem fs = LocalFileSystem(); - final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); - Directory comparisonRoot; - - if (suffix != null) { - comparisonRoot = fs.systemTempDirectory.createTempSync(suffix); - } else { - comparisonRoot = flutterRoot.childDirectory( - fs.path.join( - 'bin', - 'cache', - 'pkg', - 'skia_goldens', - ) - ); - } - - final Directory testDirectory = fs.directory(defaultComparator.basedir); - final String testDirectoryRelativePath = fs.path.relative( - testDirectory.path, - from: flutterRoot.path, - ); - return comparisonRoot.childDirectory(testDirectoryRelativePath); - } - - /// Returns the golden [File] identified by the given [Uri]. - @protected - File getGoldenFile(Uri uri) { - final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); - return goldenFile; - } - - /// Prepends the golden URL with the library name that encloses the current - /// test. - Uri _addPrefix(Uri golden) { - // Ensure the Uri ends in .png as the SkiaClient expects - assert( - golden.toString().split('.').last == 'png', - 'Golden files in the Flutter framework must end with the file extension ' - '.png.' - ); - return Uri.parse([ - if (namePrefix != null) - namePrefix!, - basedir.pathSegments[basedir.pathSegments.length - 2], - golden.toString(), - ].join('.')); - } -} - -/// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in -/// post-submit. -/// -/// For testing across all platforms, the [SkiaGoldClient] is used to upload -/// images for framework-related golden tests and process results. -/// -/// See also: -/// -/// * [GoldenFileComparator], the abstract class that -/// [FlutterGoldenFileComparator] implements. -/// * [FlutterPreSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images before changes are -/// merged into the master branch. -/// * [FlutterLocalFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images locally on your -/// current machine. -class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { - /// Creates a [FlutterPostSubmitFileComparator] that will test golden file - /// images against Skia Gold. - /// - /// The [fs] and [platform] parameters are useful in tests, where the default - /// file system and platform can be replaced by mock instances. - FlutterPostSubmitFileComparator( - super.basedir, - super.skiaClient, { - super.fs, - super.platform, - super.namePrefix, - }); - - /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative - /// path resolution of the default [goldenFileComparator]. - /// - /// The [goldens] and [defaultComparator] parameters are visible for testing - /// purposes only. - static Future fromDefaultComparator( - final Platform platform, { - SkiaGoldClient? goldens, - LocalFileComparator? defaultComparator, - String? namePrefix, - }) async { - - defaultComparator ??= goldenFileComparator as LocalFileComparator; - final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( - defaultComparator, - platform, - suffix: 'flutter_goldens_postsubmit.', - ); - baseDirectory.createSync(recursive: true); - - goldens ??= SkiaGoldClient(baseDirectory); - await goldens.auth(); - return FlutterPostSubmitFileComparator(baseDirectory.uri, goldens, namePrefix: namePrefix); - } - - @override - Future compare(Uint8List imageBytes, Uri golden) async { - await skiaClient.imgtestInit(); - golden = _addPrefix(golden); - await update(golden, imageBytes); - final File goldenFile = getGoldenFile(golden); - - return skiaClient.imgtestAdd(golden.path, goldenFile); - } - - /// Decides based on the current environment if goldens tests should be - /// executed through Skia Gold. - static bool isAvailableForEnvironment(Platform platform) { - final bool luciPostSubmit = platform.environment.containsKey('SWARMING_TASK_ID') - && platform.environment.containsKey('GOLDCTL') - // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. - && !platform.environment.containsKey('GOLD_TRYJOB'); - - return luciPostSubmit; - } -} - -/// A [FlutterGoldenFileComparator] for testing golden images before changes are -/// merged into the master branch. The comparator executes tryjobs using the -/// [SkiaGoldClient]. -/// -/// See also: -/// -/// * [GoldenFileComparator], the abstract class that -/// [FlutterGoldenFileComparator] implements. -/// * [FlutterPostSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold -/// dashboard in post-submit. -/// * [FlutterLocalFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images locally on your -/// current machine. -class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { - /// Creates a [FlutterPreSubmitFileComparator] that will test golden file - /// images against baselines requested from Flutter Gold. - /// - /// The [fs] and [platform] parameters are useful in tests, where the default - /// file system and platform can be replaced by mock instances. - FlutterPreSubmitFileComparator( - super.basedir, - super.skiaClient, { - super.fs, - super.platform, - super.namePrefix, - }); - - /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the - /// relative path resolution of the default [goldenFileComparator]. - /// - /// The [goldens] and [defaultComparator] parameters are visible for testing - /// purposes only. - static Future fromDefaultComparator( - final Platform platform, { - SkiaGoldClient? goldens, - LocalFileComparator? defaultComparator, - Directory? testBasedir, - String? namePrefix, - }) async { - - defaultComparator ??= goldenFileComparator as LocalFileComparator; - final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory( - defaultComparator, - platform, - suffix: 'flutter_goldens_presubmit.', - ); - - if (!baseDirectory.existsSync()) { - baseDirectory.createSync(recursive: true); - } - - goldens ??= SkiaGoldClient(baseDirectory); - - await goldens.auth(); - return FlutterPreSubmitFileComparator( - baseDirectory.uri, - goldens, platform: platform, - namePrefix: namePrefix, - ); - } - - @override - Future compare(Uint8List imageBytes, Uri golden) async { - await skiaClient.tryjobInit(); - golden = _addPrefix(golden); - await update(golden, imageBytes); - final File goldenFile = getGoldenFile(golden); - - await skiaClient.tryjobAdd(golden.path, goldenFile); - - // This will always return true since golden file test failures are managed - // in pre-submit checks by the flutter-gold status check. - return true; - } - - /// Decides based on the current environment if goldens tests should be - /// executed as pre-submit tests with Skia Gold. - static bool isAvailableForEnvironment(Platform platform) { - final bool luciPreSubmit = platform.environment.containsKey('SWARMING_TASK_ID') - && platform.environment.containsKey('GOLDCTL') - && platform.environment.containsKey('GOLD_TRYJOB'); - return luciPreSubmit; - } -} - -/// A [FlutterGoldenFileComparator] for testing conditions that do not execute -/// golden file tests. -/// -/// Currently, this comparator is used on Cirrus, or in Luci environments when executing tests -/// outside of the flutter/flutter repository. -/// -/// See also: -/// -/// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator] -/// that tests golden images through Skia Gold. -/// * [FlutterPreSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images before changes are -/// merged into the master branch. -/// * [FlutterLocalFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images locally on your -/// current machine. -class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { - /// Creates a [FlutterSkippingFileComparator] that will skip tests that - /// are not in the right environment for golden file testing. - FlutterSkippingFileComparator( - super.basedir, - super.skiaClient, - this.reason, { - super.namePrefix, - }); - - /// Describes the reason for using the [FlutterSkippingFileComparator]. - /// - /// Cannot be null. - final String reason; - - /// Creates a new [FlutterSkippingFileComparator] that mirrors the - /// relative path resolution of the default [goldenFileComparator]. - static FlutterSkippingFileComparator fromDefaultComparator( - String reason, { - LocalFileComparator? defaultComparator, - String? namePrefix, - }) { - defaultComparator ??= goldenFileComparator as LocalFileComparator; - const FileSystem fs = LocalFileSystem(); - final Uri basedir = defaultComparator.basedir; - final SkiaGoldClient skiaClient = SkiaGoldClient(fs.directory(basedir)); - return FlutterSkippingFileComparator(basedir, skiaClient, reason, namePrefix: namePrefix); - } - - @override - Future compare(Uint8List imageBytes, Uri golden) async { - // Ideally we would use markTestSkipped here but in some situations, - // comparators are called outside of tests. - // See also: https://github.com/flutter/flutter/issues/91285 - // ignore: avoid_print - print('Skipping "$golden" test: $reason'); - return true; - } - - @override - Future update(Uri golden, Uint8List imageBytes) async {} - - /// Decides, based on the current environment, if this comparator should be - /// used. - /// - /// If we are in a CI environment, LUCI or Cirrus, but are not using the other - /// comparators, we skip. - static bool isAvailableForEnvironment(Platform platform) { - return platform.environment.containsKey('SWARMING_TASK_ID') - // Some builds are still being run on Cirrus, we should skip these. - || platform.environment.containsKey('CIRRUS_CI'); - } -} - -/// A [FlutterGoldenFileComparator] for testing golden images locally on your -/// current machine. -/// -/// This comparator utilizes the [SkiaGoldClient] to request baseline images for -/// the given device under test for comparison. This comparator is initialized -/// when conditions for all other [FlutterGoldenFileComparators] have not been -/// met, see the `isAvailableForEnvironment` method for each one listed below. -/// -/// The [FlutterLocalFileComparator] is intended to run on local machines and -/// serve as a smoke test during development. As such, it will not be able to -/// detect unintended changes on environments other than the currently executing -/// machine, until they are tested using the [FlutterPreSubmitFileComparator]. -/// -/// See also: -/// -/// * [GoldenFileComparator], the abstract class that -/// [FlutterGoldenFileComparator] implements. -/// * [FlutterPostSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold -/// dashboard. -/// * [FlutterPreSubmitFileComparator], another -/// [FlutterGoldenFileComparator] that tests golden images before changes are -/// merged into the master branch. -/// * [FlutterSkippingFileComparator], another -/// [FlutterGoldenFileComparator] that controls post-submit testing -/// conditions that do not execute golden file tests. -class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput { - /// Creates a [FlutterLocalFileComparator] that will test golden file - /// images against baselines requested from Flutter Gold. - /// - /// The [fs] and [platform] parameters are useful in tests, where the default - /// file system and platform can be replaced by mock instances. - FlutterLocalFileComparator( - super.basedir, - super.skiaClient, { - super.fs, - super.platform, - }); - - /// Creates a new [FlutterLocalFileComparator] that mirrors the - /// relative path resolution of the default [goldenFileComparator]. - /// - /// The [goldens], [defaultComparator], and [baseDirectory] parameters are - /// visible for testing purposes only. - static Future fromDefaultComparator( - final Platform platform, { - SkiaGoldClient? goldens, - LocalFileComparator? defaultComparator, - Directory? baseDirectory, - }) async { - defaultComparator ??= goldenFileComparator as LocalFileComparator; - baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( - defaultComparator, - platform, - ); - - if(!baseDirectory.existsSync()) { - baseDirectory.createSync(recursive: true); - } - - goldens ??= SkiaGoldClient(baseDirectory); - try { - // Check if we can reach Gold. - await goldens.getExpectationForTest(''); - } on io.OSError catch (_) { - return FlutterSkippingFileComparator( - baseDirectory.uri, - goldens, - 'OSError occurred, could not reach Gold. ' - 'Switching to FlutterSkippingGoldenFileComparator.', - ); - } on io.SocketException catch (_) { - return FlutterSkippingFileComparator( - baseDirectory.uri, - goldens, - 'SocketException occurred, could not reach Gold. ' - 'Switching to FlutterSkippingGoldenFileComparator.', - ); - } - - return FlutterLocalFileComparator(baseDirectory.uri, goldens); - } - - @override - Future compare(Uint8List imageBytes, Uri golden) async { - golden = _addPrefix(golden); - final String testName = skiaClient.cleanTestName(golden.path); - late String? testExpectation; - testExpectation = await skiaClient.getExpectationForTest(testName); - - if (testExpectation == null || testExpectation.isEmpty) { - // There is no baseline for this test. - // Ideally we would use markTestSkipped here but in some situations, - // comparators are called outside of tests. - // See also: https://github.com/flutter/flutter/issues/91285 - // ignore: avoid_print - print( - 'No expectations provided by Skia Gold for test: $golden. ' - 'This may be a new test. If this is an unexpected result, check ' - 'https://flutter-gold.skia.org.\n' - 'Validate image output found at $basedir' - ); - update(golden, imageBytes); - return true; - } - - ComparisonResult result; - final List goldenBytes = await skiaClient.getImageBytes(testExpectation); - - result = await GoldenFileComparator.compareLists( - imageBytes, - goldenBytes, - ); - - if (result.passed) { - return true; - } - - final String error = await generateFailureOutput(result, golden, basedir); - throw FlutterError(error); - } -} +export 'src/flaky_goldens.dart' show expectFlakyGolden; +export 'src/flutter_goldens_io.dart' if (dart.library.js_util) 'src/flutter_goldens_web.dart' + show processBrowserCommand, testExecutable; diff --git a/packages/flutter_goldens/lib/src/flaky_goldens.dart b/packages/flutter_goldens/lib/src/flaky_goldens.dart new file mode 100644 index 000000000000..9142793f6c48 --- /dev/null +++ b/packages/flutter_goldens/lib/src/flaky_goldens.dart @@ -0,0 +1,98 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// flutter_ignore_for_file: golden_tag (see analyze.dart) + +import 'package:flutter_test/flutter_test.dart'; + +/// Similar to [matchesGoldenFile] but specialized for Flutter's own tests when +/// they are flaky. +/// +/// Asserts that a [Finder], [Future], or [ui.Image] - the [key] - +/// matches the golden image file identified by [goldenFile]. +/// +/// For the case of a [Finder], the [Finder] must match exactly one widget and +/// the rendered image of the first [RepaintBoundary] ancestor of the widget is +/// treated as the image for the widget. As such, you may choose to wrap a test +/// widget in a [RepaintBoundary] to specify a particular focus for the test. +/// +/// The [goldenFile] may be either a [Uri] or a [String] representation of a URL. +/// +/// Flaky golden file tests are always uploaded to Skia Gold for manual +/// inspection. This allows contributors to validate when a test is no longer +/// flaky by visiting https://flutter-gold.skia.org/list, +/// and clicking on the respective golden test name. The UI will show the +/// history of generated goldens over time. Each unique golden gets a unique +/// color. If the color is the same for all commits in the recent history, the +/// golden is likely no longer flaky and the standard [matchesGoldenFile] can be +/// used in the given test. If the color changes from commit to commit then it +/// is still flaky. +Future expectFlakyGolden(Object key, String goldenFile) { + if (isBrowser) { + _setFlakyForWeb(); + } else { + _setFlakyForIO(); + } + return expectLater(key, matchesGoldenFile(goldenFile)); +} + +void _setFlakyForWeb() { + assert( + webGoldenComparator is FlakyGoldenMixin, + 'expectFlakyGolden can only be used with a comparator with the FlakyGoldenMixin ' + 'but found ${webGoldenComparator.runtimeType}.' + ); + (webGoldenComparator as FlakyGoldenMixin).enableFlakyMode(); +} + +void _setFlakyForIO() { + assert( + goldenFileComparator is FlakyGoldenMixin, + 'expectFlakyGolden can only be used with a comparator with the FlakyGoldenMixin ' + 'but found ${goldenFileComparator.runtimeType}.' + ); + (goldenFileComparator as FlakyGoldenMixin).enableFlakyMode(); +} + +/// Allows flaky test handling for the Flutter framework. +/// +/// Mixed in with the [FlutterGoldenFileComparator] and +/// [_FlutterWebGoldenComparator]. +mixin FlakyGoldenMixin { + /// Whether this comparator allows flaky goldens. + /// + /// If set to true, concrete implementations of this class are expected to + /// generate the golden and submit it for review, but not fail the test. + bool _isFlakyModeEnabled = false; + + /// Puts this comparator into flaky comparison mode. + /// + /// After calling this method the next invocation of [compare] will allow + /// incorrect golden to pass the check. + /// + /// Concrete implementations of [compare] must call [getAndResetFlakyMode] so + /// that subsequent tests can run in non-flaky mode. If a subsequent test + /// needs to run in a flaky mode, it must call this method again. + void enableFlakyMode() { + assert( + !_isFlakyModeEnabled, + 'Test is already marked as flaky. Call `getAndResetFlakyMode` to reset the ' + 'flag before calling this method again.', + ); + _isFlakyModeEnabled = true; + } + + /// Returns whether flaky comparison mode was enabled via [enableFlakyMode], + /// and if it was, resets the comparator back to non-flaky mode. + bool getAndResetFlakyMode() { + if (!_isFlakyModeEnabled) { + // Not in flaky mode. Nothing to do. + return false; + } + + // In flaky mode. Reset it and return true. + _isFlakyModeEnabled = false; + return true; + } +} diff --git a/packages/flutter_goldens/lib/src/flutter_goldens_io.dart b/packages/flutter_goldens/lib/src/flutter_goldens_io.dart new file mode 100644 index 000000000000..b64ebb7368af --- /dev/null +++ b/packages/flutter_goldens/lib/src/flutter_goldens_io.dart @@ -0,0 +1,611 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async' show FutureOr; +import 'dart:convert' show jsonEncode; +import 'dart:io' as io show File, OSError, SocketException, stdout; + +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_goldens_client/skia_client.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:platform/platform.dart'; + +import 'flaky_goldens.dart'; + +export 'package:flutter_goldens_client/skia_client.dart'; + +// If you are here trying to figure out how to use golden files in the Flutter +// repo itself, consider reading this wiki page: +// https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package%3Aflutter + +const String _kFlutterRootKey = 'FLUTTER_ROOT'; + +/// Main method that can be used in a `flutter_test_config.dart` file to set +/// [goldenFileComparator] to an instance of [FlutterGoldenFileComparator] that +/// works for the current test. _Which_ FlutterGoldenFileComparator is +/// instantiated is based on the current testing environment. +/// +/// When set, the `namePrefix` is prepended to the names of all gold images. +Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async { + const Platform platform = LocalPlatform(); + if (FlutterPostSubmitFileComparator.isAvailableForEnvironment(platform)) { + goldenFileComparator = await FlutterPostSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix); + } else if (FlutterPreSubmitFileComparator.isAvailableForEnvironment(platform)) { + goldenFileComparator = await FlutterPreSubmitFileComparator.fromDefaultComparator(platform, namePrefix: namePrefix); + } else if (FlutterSkippingFileComparator.isAvailableForEnvironment(platform)) { + goldenFileComparator = FlutterSkippingFileComparator.fromDefaultComparator( + 'Golden file testing is not executed on Cirrus, or LUCI environments outside of flutter/flutter.', + namePrefix: namePrefix + ); + } else { + goldenFileComparator = await FlutterLocalFileComparator.fromDefaultComparator(platform); + } + + await testMain(); +} + +/// Processes golden check commands sent from the browser process. +/// +/// When running browser tests, goldens are not generated within the app itself +/// due to browser restrictions. Instead, when a test calls [expectFlakyGolden] +/// the browser sends a [command] to a host process. This function handles the +/// command. +/// +/// This custom command handler is used for Flutter's own goldens. It +/// understands the "isFlaky" property, a boolean encoded in the command as a +/// custom command property. If true, uses a golden comparator that submits the +/// golden, but does not fail the test, allowing manual inspection in the Skia +/// Gold UI and verification of fixes. +/// +/// See also: +/// * [FlakyWebGoldenComparator], which implements custom browser-side logic. +Future processBrowserCommand(dynamic command) async { + if (command is Map) { + final io.File imageFile = io.File(command['imageFile'] as String); + final Uri goldenKey = Uri.parse(command['key'] as String); + final bool update = command['update'] as bool; + final Map customProperties = (command['customProperties'] as Map?) ?? const {}; + final bool isFlaky = (customProperties['isFlaky'] as bool?) ?? false; + final Uint8List bytes = await io.File(imageFile.path).readAsBytes(); + if (update) { + await goldenFileComparator.update(goldenKey, bytes); + io.stdout.writeln(jsonEncode({'success': true})); + } else { + try { + assert( + goldenFileComparator is FlutterGoldenFileComparator, + 'matchesFlutterGolden can only be used with FlutterGoldenFileComparator ' + 'but found ${goldenFileComparator.runtimeType}.' + ); + + if (isFlaky) { + (goldenFileComparator as FlutterGoldenFileComparator).enableFlakyMode(); + } + final bool success = await goldenFileComparator.compare(bytes, goldenKey); + io.stdout.writeln(jsonEncode({'success': success})); + } on Exception catch (exception) { + io.stdout.writeln(jsonEncode({'success': false, 'message': '$exception'})); + } + } + } else { + io.stdout.writeln('object type is not right'); + } +} + +/// Abstract base class golden file comparator specific to the `flutter/flutter` +/// repository. +/// +/// Golden file testing for the `flutter/flutter` repository is handled by three +/// different [FlutterGoldenFileComparator]s, depending on the current testing +/// environment. +/// +/// * The [FlutterPostSubmitFileComparator] is utilized during post-submit +/// testing, after a pull request has landed on the master branch. This +/// comparator uses the [SkiaGoldClient] and the `goldctl` tool to upload +/// tests to the [Flutter Gold dashboard](https://flutter-gold.skia.org). +/// Flutter Gold manages the master golden files for the `flutter/flutter` +/// repository. +/// +/// * The [FlutterPreSubmitFileComparator] is utilized in pre-submit testing, +/// before a pull request lands on the master branch. This +/// comparator uses the [SkiaGoldClient] to execute tryjobs, allowing +/// contributors to view and check in visual differences before landing the +/// change. +/// +/// * The [FlutterLocalFileComparator] is used for local development testing. +/// This comparator will use the [SkiaGoldClient] to request baseline images +/// from [Flutter Gold](https://flutter-gold.skia.org) and manually compare +/// pixels. If a difference is detected, this comparator will +/// generate failure output illustrating the found difference. If a baseline +/// is not found for a given test image, it will consider it a new test and +/// output the new image for verification. +/// +/// The [FlutterSkippingFileComparator] is utilized to skip tests outside +/// of the appropriate environments described above. Currently, some Luci +/// environments do not execute golden file testing, and as such do not require +/// a comparator. This comparator is also used when an internet connection is +/// unavailable. +abstract class FlutterGoldenFileComparator extends GoldenFileComparator with FlakyGoldenMixin { + /// Creates a [FlutterGoldenFileComparator] that will resolve golden file + /// URIs relative to the specified [basedir], and retrieve golden baselines + /// using the [skiaClient]. The [basedir] is used for writing and accessing + /// information and files for interacting with the [skiaClient]. When testing + /// locally, the [basedir] will also contain any diffs from failed tests, or + /// goldens generated from newly introduced tests. + /// + /// The [fs] and [platform] parameters are useful in tests, where the default + /// file system and platform can be replaced by mock instances. + @visibleForTesting + FlutterGoldenFileComparator( + this.basedir, + this.skiaClient, { + this.fs = const LocalFileSystem(), + this.platform = const LocalPlatform(), + this.namePrefix, + }); + + /// The directory to which golden file URIs will be resolved in [compare] and + /// [update], cannot be null. + final Uri basedir; + + /// A client for uploading image tests and making baseline requests to the + /// Flutter Gold Dashboard, cannot be null. + final SkiaGoldClient skiaClient; + + /// The file system used to perform file access. + @visibleForTesting + final FileSystem fs; + + /// A wrapper for the [dart:io.Platform] API. + @visibleForTesting + final Platform platform; + + /// The prefix that is added to all golden names. + final String? namePrefix; + + @override + Future update(Uri golden, Uint8List imageBytes) async { + final File goldenFile = getGoldenFile(golden); + await goldenFile.parent.create(recursive: true); + await goldenFile.writeAsBytes(imageBytes, flush: true); + } + + @override + Uri getTestUri(Uri key, int? version) => key; + + /// Calculate the appropriate basedir for the current test context. + /// + /// The optional [suffix] argument is used by the + /// [FlutterPostSubmitFileComparator] and the [FlutterPreSubmitFileComparator]. + /// These [FlutterGoldenFileComparators] randomize their base directories to + /// maintain thread safety while using the `goldctl` tool. + @protected + @visibleForTesting + static Directory getBaseDirectory( + LocalFileComparator defaultComparator, + Platform platform, { + String? suffix, + }) { + const FileSystem fs = LocalFileSystem(); + final Directory flutterRoot = fs.directory(platform.environment[_kFlutterRootKey]); + Directory comparisonRoot; + + if (suffix != null) { + comparisonRoot = fs.systemTempDirectory.createTempSync(suffix); + } else { + comparisonRoot = flutterRoot.childDirectory( + fs.path.join( + 'bin', + 'cache', + 'pkg', + 'skia_goldens', + ) + ); + } + + final Directory testDirectory = fs.directory(defaultComparator.basedir); + final String testDirectoryRelativePath = fs.path.relative( + testDirectory.path, + from: flutterRoot.path, + ); + return comparisonRoot.childDirectory(testDirectoryRelativePath); + } + + /// Returns the golden [File] identified by the given [Uri]. + @protected + File getGoldenFile(Uri uri) { + final File goldenFile = fs.directory(basedir).childFile(fs.file(uri).path); + return goldenFile; + } + + /// Prepends the golden URL with the library name that encloses the current + /// test. + Uri _addPrefix(Uri golden) { + // Ensure the Uri ends in .png as the SkiaClient expects + assert( + golden.toString().split('.').last == 'png', + 'Golden files in the Flutter framework must end with the file extension ' + '.png.' + ); + return Uri.parse([ + if (namePrefix != null) + namePrefix!, + basedir.pathSegments[basedir.pathSegments.length - 2], + golden.toString(), + ].join('.')); + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images with Skia Gold in +/// post-submit. +/// +/// For testing across all platforms, the [SkiaGoldClient] is used to upload +/// images for framework-related golden tests and process results. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterPostSubmitFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterPostSubmitFileComparator] that will test golden file + /// images against Skia Gold. + /// + /// The [fs] and [platform] parameters are useful in tests, where the default + /// file system and platform can be replaced by mock instances. + FlutterPostSubmitFileComparator( + super.basedir, + super.skiaClient, { + super.fs, + super.platform, + super.namePrefix, + }); + + /// Creates a new [FlutterPostSubmitFileComparator] that mirrors the relative + /// path resolution of the default [goldenFileComparator]. + /// + /// The [goldens] and [defaultComparator] parameters are visible for testing + /// purposes only. + static Future fromDefaultComparator( + final Platform platform, { + SkiaGoldClient? goldens, + LocalFileComparator? defaultComparator, + String? namePrefix, + }) async { + defaultComparator ??= goldenFileComparator as LocalFileComparator; + final Directory baseDirectory = FlutterGoldenFileComparator.getBaseDirectory( + defaultComparator, + platform, + suffix: 'flutter_goldens_postsubmit.', + ); + baseDirectory.createSync(recursive: true); + + goldens ??= SkiaGoldClient(baseDirectory); + await goldens.auth(); + return FlutterPostSubmitFileComparator(baseDirectory.uri, goldens, namePrefix: namePrefix); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + final bool isFlaky = getAndResetFlakyMode(); + await skiaClient.imgtestInit(isFlaky: isFlaky); + golden = _addPrefix(golden); + await update(golden, imageBytes); + final File goldenFile = getGoldenFile(golden); + + return skiaClient.imgtestAdd(golden.path, goldenFile, isFlaky: isFlaky); + } + + /// Decides based on the current environment if goldens tests should be + /// executed through Skia Gold. + static bool isAvailableForEnvironment(Platform platform) { + final bool luciPostSubmit = platform.environment.containsKey('SWARMING_TASK_ID') + && platform.environment.containsKey('GOLDCTL') + // Luci tryjob environments contain this value to inform the [FlutterPreSubmitComparator]. + && !platform.environment.containsKey('GOLD_TRYJOB'); + + return luciPostSubmit; + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images before changes are +/// merged into the master branch. The comparator executes tryjobs using the +/// [SkiaGoldClient]. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPostSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold +/// dashboard in post-submit. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterPreSubmitFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterPreSubmitFileComparator] that will test golden file + /// images against baselines requested from Flutter Gold. + /// + /// The [fs] and [platform] parameters are useful in tests, where the default + /// file system and platform can be replaced by mock instances. + FlutterPreSubmitFileComparator( + super.basedir, + super.skiaClient, { + super.fs, + super.platform, + super.namePrefix, + }); + + /// Creates a new [FlutterPreSubmitFileComparator] that mirrors the + /// relative path resolution of the default [goldenFileComparator]. + /// + /// The [goldens] and [defaultComparator] parameters are visible for testing + /// purposes only. + static Future fromDefaultComparator( + final Platform platform, { + SkiaGoldClient? goldens, + LocalFileComparator? defaultComparator, + Directory? testBasedir, + String? namePrefix, + }) async { + defaultComparator ??= goldenFileComparator as LocalFileComparator; + final Directory baseDirectory = testBasedir ?? FlutterGoldenFileComparator.getBaseDirectory( + defaultComparator, + platform, + suffix: 'flutter_goldens_presubmit.', + ); + + if (!baseDirectory.existsSync()) { + baseDirectory.createSync(recursive: true); + } + + goldens ??= SkiaGoldClient(baseDirectory); + + await goldens.auth(); + return FlutterPreSubmitFileComparator( + baseDirectory.uri, + goldens, platform: platform, + namePrefix: namePrefix, + ); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + final bool isFlaky = getAndResetFlakyMode(); + await skiaClient.tryjobInit(isFlaky: isFlaky); + golden = _addPrefix(golden); + await update(golden, imageBytes); + final File goldenFile = getGoldenFile(golden); + + await skiaClient.tryjobAdd(golden.path, goldenFile, isFlaky: isFlaky); + + // This will always return true since golden file test failures are managed + // in pre-submit checks by the flutter-gold status check. + return true; + } + + /// Decides based on the current environment if goldens tests should be + /// executed as pre-submit tests with Skia Gold. + static bool isAvailableForEnvironment(Platform platform) { + final bool luciPreSubmit = platform.environment.containsKey('SWARMING_TASK_ID') + && platform.environment.containsKey('GOLDCTL') + && platform.environment.containsKey('GOLD_TRYJOB'); + return luciPreSubmit; + } +} + +/// A [FlutterGoldenFileComparator] for testing conditions that do not execute +/// golden file tests. +/// +/// Currently, this comparator is used on Cirrus, or in Luci environments when executing tests +/// outside of the flutter/flutter repository. +/// +/// See also: +/// +/// * [FlutterPostSubmitFileComparator], another [FlutterGoldenFileComparator] +/// that tests golden images through Skia Gold. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterLocalFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images locally on your +/// current machine. +class FlutterSkippingFileComparator extends FlutterGoldenFileComparator { + /// Creates a [FlutterSkippingFileComparator] that will skip tests that + /// are not in the right environment for golden file testing. + FlutterSkippingFileComparator( + super.basedir, + super.skiaClient, + this.reason, { + super.namePrefix, + }); + + /// Describes the reason for using the [FlutterSkippingFileComparator]. + /// + /// Cannot be null. + final String reason; + + /// Creates a new [FlutterSkippingFileComparator] that mirrors the + /// relative path resolution of the default [goldenFileComparator]. + static FlutterSkippingFileComparator fromDefaultComparator( + String reason, { + LocalFileComparator? defaultComparator, + String? namePrefix, + bool isFlaky = false, + }) { + defaultComparator ??= goldenFileComparator as LocalFileComparator; + const FileSystem fs = LocalFileSystem(); + final Uri basedir = defaultComparator.basedir; + final SkiaGoldClient skiaClient = SkiaGoldClient(fs.directory(basedir)); + return FlutterSkippingFileComparator(basedir, skiaClient, reason, namePrefix: namePrefix); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + // Ideally we would use markTestSkipped here but in some situations, + // comparators are called outside of tests. + // See also: https://github.com/flutter/flutter/issues/91285 + // ignore: avoid_print + getAndResetFlakyMode(); + // ignore: avoid_print + print('Skipping "$golden" test: $reason'); + return true; + } + + @override + Future update(Uri golden, Uint8List imageBytes) async {} + + /// Decides, based on the current environment, if this comparator should be + /// used. + /// + /// If we are in a CI environment, LUCI or Cirrus, but are not using the other + /// comparators, we skip. + static bool isAvailableForEnvironment(Platform platform) { + return platform.environment.containsKey('SWARMING_TASK_ID') + // Some builds are still being run on Cirrus, we should skip these. + || platform.environment.containsKey('CIRRUS_CI'); + } +} + +/// A [FlutterGoldenFileComparator] for testing golden images locally on your +/// current machine. +/// +/// This comparator utilizes the [SkiaGoldClient] to request baseline images for +/// the given device under test for comparison. This comparator is initialized +/// when conditions for all other [FlutterGoldenFileComparators] have not been +/// met, see the `isAvailableForEnvironment` method for each one listed below. +/// +/// The [FlutterLocalFileComparator] is intended to run on local machines and +/// serve as a smoke test during development. As such, it will not be able to +/// detect unintended changes on environments other than the currently executing +/// machine, until they are tested using the [FlutterPreSubmitFileComparator]. +/// +/// See also: +/// +/// * [GoldenFileComparator], the abstract class that +/// [FlutterGoldenFileComparator] implements. +/// * [FlutterPostSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that uploads tests to the Skia Gold +/// dashboard. +/// * [FlutterPreSubmitFileComparator], another +/// [FlutterGoldenFileComparator] that tests golden images before changes are +/// merged into the master branch. +/// * [FlutterSkippingFileComparator], another +/// [FlutterGoldenFileComparator] that controls post-submit testing +/// conditions that do not execute golden file tests. +class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalComparisonOutput { + /// Creates a [FlutterLocalFileComparator] that will test golden file + /// images against baselines requested from Flutter Gold. + /// + /// The [fs] and [platform] parameters are useful in tests, where the default + /// file system and platform can be replaced by mock instances. + FlutterLocalFileComparator( + super.basedir, + super.skiaClient, { + super.fs, + super.platform, + }); + + /// Creates a new [FlutterLocalFileComparator] that mirrors the + /// relative path resolution of the default [goldenFileComparator]. + /// + /// The [goldens], [defaultComparator], and [baseDirectory] parameters are + /// visible for testing purposes only. + static Future fromDefaultComparator( + final Platform platform, { + SkiaGoldClient? goldens, + LocalFileComparator? defaultComparator, + Directory? baseDirectory, + }) async { + defaultComparator ??= goldenFileComparator as LocalFileComparator; + baseDirectory ??= FlutterGoldenFileComparator.getBaseDirectory( + defaultComparator, + platform, + ); + + if(!baseDirectory.existsSync()) { + baseDirectory.createSync(recursive: true); + } + + goldens ??= SkiaGoldClient(baseDirectory); + try { + // Check if we can reach Gold. + await goldens.getExpectationForTest(''); + } on io.OSError catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'OSError occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + ); + } on io.SocketException catch (_) { + return FlutterSkippingFileComparator( + baseDirectory.uri, + goldens, + 'SocketException occurred, could not reach Gold. ' + 'Switching to FlutterSkippingGoldenFileComparator.', + ); + } + + return FlutterLocalFileComparator(baseDirectory.uri, goldens); + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + final bool isFlaky = getAndResetFlakyMode(); + golden = _addPrefix(golden); + final String testName = skiaClient.cleanTestName(golden.path); + late String? testExpectation; + testExpectation = await skiaClient.getExpectationForTest(testName); + + if (testExpectation == null || testExpectation.isEmpty) { + // There is no baseline for this test. + // Ideally we would use markTestSkipped here but in some situations, + // comparators are called outside of tests. + // See also: https://github.com/flutter/flutter/issues/91285 + // ignore: avoid_print + print( + 'No expectations provided by Skia Gold for test: $golden. ' + 'This may be a new test. If this is an unexpected result, check ' + 'https://flutter-gold.skia.org.\n' + 'Validate image output found at $basedir' + ); + update(golden, imageBytes); + return true; + } + + ComparisonResult result; + final List goldenBytes = await skiaClient.getImageBytes(testExpectation); + + result = await GoldenFileComparator.compareLists( + imageBytes, + goldenBytes, + ); + + if (result.passed) { + return true; + } + + final String error = await generateFailureOutput(result, golden, basedir); + if (!isFlaky) { + throw FlutterError(error); + } else { + // The test was marked as flaky. Do not fail the test. + // TODO(yjbanov): there's no way to communicate warnings to the caller https://github.com/flutter/flutter/issues/91285 + // ignore: avoid_print + print('Golden $golden is marked as flaky and will not fail the test.'); + // ignore: avoid_print + print(error); + } + + return true; + } +} diff --git a/packages/flutter_goldens/lib/src/flutter_goldens_web.dart b/packages/flutter_goldens/lib/src/flutter_goldens_web.dart new file mode 100644 index 000000000000..27641d3ac67c --- /dev/null +++ b/packages/flutter_goldens/lib/src/flutter_goldens_web.dart @@ -0,0 +1,71 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async' show FutureOr; +import 'dart:convert' show json; +import 'dart:html' as html; + +import 'package:flutter_test/flutter_test.dart'; + +import 'flaky_goldens.dart'; + +export 'package:flutter_goldens_client/skia_client.dart'; + +/// Wraps a web test, supplying a custom comparator that supports flaky goldens. +Future testExecutable(FutureOr Function() testMain, {String? namePrefix}) async { + webGoldenComparator = FlutterWebGoldenComparator(webTestUri); + await testMain(); +} + +/// See the io implementation of this function. +Future processBrowserCommand(dynamic command) async { + throw UnimplementedError('processBrowserCommand is not used inside the browser'); +} + +/// Same as [DefaultWebGoldenComparator] but supports flaky golden checks. +class FlutterWebGoldenComparator extends WebGoldenComparator with FlakyGoldenMixin { + /// Creates a new [FlutterWebGoldenComparator] for the specified [testUri]. + /// + /// Golden file keys will be interpreted as file paths relative to the + /// directory in which [testUri] resides. + /// + /// The [testUri] URL must represent a file. + FlutterWebGoldenComparator(this.testUri); + + /// The test file currently being executed. + /// + /// Golden file keys will be interpreted as file paths relative to the + /// directory in which this file resides. + Uri testUri; + + @override + Future compare(double width, double height, Uri golden) async { + final bool isFlaky = getAndResetFlakyMode(); + final String key = golden.toString(); + final html.HttpRequest request = await html.HttpRequest.request( + 'flutter_goldens', + method: 'POST', + sendData: json.encode({ + 'testUri': testUri.toString(), + 'key': key, + 'width': width.round(), + 'height': height.round(), + 'customProperties': { + 'isFlaky': isFlaky, + }, + }), + ); + final String response = request.response as String; + if (response == 'true') { + return true; + } + fail(response); + } + + @override + Future update(double width, double height, Uri golden) async { + // Update is handled on the server side, just use the same logic here + await compare(width, height, golden); + } +} diff --git a/packages/flutter_goldens/test/flaky_goldens_test.dart b/packages/flutter_goldens/test/flaky_goldens_test.dart new file mode 100644 index 000000000000..e3eedfa2b7ac --- /dev/null +++ b/packages/flutter_goldens/test/flaky_goldens_test.dart @@ -0,0 +1,52 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// See also dev/automated_tests/flutter_test/flutter_gold_test.dart + +// flutter_ignore_for_file: golden_tag (see analyze.dart) + +import 'package:flutter_goldens/flutter_goldens.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'utils/fakes.dart'; + +void main() { + test('Sets flaky flag', () { + final FakeFlakyLocalFileComparator comparator = FakeFlakyLocalFileComparator(); + // Is not flaky + expect(comparator.getAndResetFlakyMode(), isFalse); + comparator.enableFlakyMode(); + // Flaky was set + expect(comparator.getAndResetFlakyMode(), isTrue); + // Flaky was unset + expect(comparator.getAndResetFlakyMode(), isFalse); + }); + + test('Asserts when comparator is missing mixin', (){ + final GoldenFileComparator oldComparator = goldenFileComparator; + goldenFileComparator = FakeLocalFileComparator(); + expect( + () { + expect( + expectFlakyGolden([0, 1, 2, 3], 'golden_file.png'), + throwsAssertionError, + ); + }, + throwsA( + isA().having((AssertionError error) => error.toString(), + 'description', contains('FlakyGoldenMixin')), + ), + ); + goldenFileComparator = oldComparator; + }); + + test('top level function sets flag', () { + final GoldenFileComparator oldComparator = goldenFileComparator; + goldenFileComparator = FakeFlakyLocalFileComparator(); + expectFlakyGolden([0, 1, 2, 3], 'golden_file.png'); + final bool wasFlaky = (goldenFileComparator as FakeFlakyLocalFileComparator).getAndResetFlakyMode(); + expect(wasFlaky, isTrue); + goldenFileComparator = oldComparator; + }); +} diff --git a/packages/flutter_goldens/test/flutter_goldens_test.dart b/packages/flutter_goldens/test/flutter_goldens_test.dart index e64198f0bc48..8e1e5cd47523 100644 --- a/packages/flutter_goldens/test/flutter_goldens_test.dart +++ b/packages/flutter_goldens/test/flutter_goldens_test.dart @@ -4,34 +4,22 @@ // See also dev/automated_tests/flutter_test/flutter_gold_test.dart -import 'dart:convert'; import 'dart:io' hide Directory; import 'package:file/file.dart'; import 'package:file/memory.dart'; import 'package:flutter/foundation.dart'; -import 'package:flutter_goldens/flutter_goldens.dart'; +import 'package:flutter_goldens/src/flutter_goldens_io.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:platform/platform.dart'; -import 'package:process/process.dart'; -import 'json_templates.dart'; +import 'utils/fakes.dart'; const String _kFlutterRoot = '/flutter'; -// 1x1 transparent pixel -const List _kTestPngBytes = [ - 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, - 1, 0, 0, 0, 1, 8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, - 120, 1, 99, 97, 0, 2, 0, 0, 25, 0, 5, 144, 240, 54, 245, 0, 0, 0, 0, 73, 69, - 78, 68, 174, 66, 96, 130, -]; - void main() { late MemoryFileSystem fs; late FakePlatform platform; - late FakeProcessManager process; - late FakeHttpClient fakeHttpClient; setUp(() { fs = MemoryFileSystem(); @@ -39,567 +27,9 @@ void main() { environment: {'FLUTTER_ROOT': _kFlutterRoot}, operatingSystem: 'macos' ); - process = FakeProcessManager(); - fakeHttpClient = FakeHttpClient(); fs.directory(_kFlutterRoot).createSync(recursive: true); }); - group('SkiaGoldClient', () { - late SkiaGoldClient skiaClient; - late Directory workDirectory; - - setUp(() { - workDirectory = fs.directory('/workDirectory') - ..createSync(recursive: true); - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - }); - - test('web HTML test', () async { - platform = FakePlatform( - environment: { - 'GOLDCTL': 'goldctl', - 'FLUTTER_ROOT': _kFlutterRoot, - 'FLUTTER_TEST_BROWSER': 'Chrome', - 'FLUTTER_WEB_RENDERER': 'html', - }, - operatingSystem: 'macos' - ); - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); - - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'add', - '--work-dir', '/workDirectory/temp', - '--test-name', 'golden_file_test', - '--png-file', '/workDirectory/temp/golden_file_test.png', - '--passfail', - '--add-test-optional-key', 'image_matching_algorithm:fuzzy', - '--add-test-optional-key', 'fuzzy_max_different_pixels:20', - '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:4', - ], - null, - ); - process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); - - expect( - await skiaClient.imgtestAdd('golden_file_test.png', goldenFile), - isTrue, - ); - }); - - test('web CanvasKit test', () async { - platform = FakePlatform( - environment: { - 'GOLDCTL': 'goldctl', - 'FLUTTER_ROOT': _kFlutterRoot, - 'FLUTTER_TEST_BROWSER': 'Chrome', - 'FLUTTER_WEB_RENDERER': 'canvaskit', - }, - operatingSystem: 'macos' - ); - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); - - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'add', - '--work-dir', '/workDirectory/temp', - '--test-name', 'golden_file_test', - '--png-file', '/workDirectory/temp/golden_file_test.png', - '--passfail', - ], - null, - ); - process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); - - expect( - await skiaClient.imgtestAdd('golden_file_test.png', goldenFile), - isTrue, - ); - }); - - test('auth performs minimal work if already authorized', () async { - final File authFile = fs.file('/workDirectory/temp/auth_opt.json') - ..createSync(recursive: true); - authFile.writeAsStringSync(authTemplate()); - process.fallbackProcessResult = ProcessResult(123, 0, '', ''); - await skiaClient.auth(); - - expect(process.workingDirectories, isEmpty); - }); - - test('gsutil is checked when authorization file is present', () async { - final File authFile = fs.file('/workDirectory/temp/auth_opt.json') - ..createSync(recursive: true); - authFile.writeAsStringSync(authTemplate(gsutil: true)); - expect( - await skiaClient.clientIsAuthorized(), - isFalse, - ); - }); - - test('throws for error state from auth', () async { - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLD_SERVICE_ACCOUNT' : 'Service Account', - 'GOLDCTL' : 'goldctl', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); - - expect( - skiaClient.auth(), - throwsException, - ); - }); - - test('throws for error state from init', () { - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - const RunInvocation gitInvocation = RunInvocation( - ['git', 'rev-parse', 'HEAD'], - '/flutter', - ); - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'init', - '--instance', 'flutter', - '--work-dir', '/workDirectory/temp', - '--commit', '12345678', - '--keys-file', '/workDirectory/keys.json', - '--failure-file', '/workDirectory/failures.json', - '--passfail', - ], - null, - ); - process.processResults[gitInvocation] = ProcessResult(12345678, 0, '12345678', ''); - process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); - - expect( - skiaClient.imgtestInit(), - throwsException, - ); - }); - - test('Only calls init once', () async { - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - const RunInvocation gitInvocation = RunInvocation( - ['git', 'rev-parse', 'HEAD'], - '/flutter', - ); - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'init', - '--instance', 'flutter', - '--work-dir', '/workDirectory/temp', - '--commit', '1234', - '--keys-file', '/workDirectory/keys.json', - '--failure-file', '/workDirectory/failures.json', - '--passfail', - ], - null, - ); - process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); - process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); - - // First call - await skiaClient.imgtestInit(); - - // Remove fake process result. - // If the init call is executed again, the fallback process will throw. - process.processResults.remove(goldctlInvocation); - - // Second call - await skiaClient.imgtestInit(); - }); - - test('Only calls tryjob init once', () async { - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB' : 'refs/pull/49815/head', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - const RunInvocation gitInvocation = RunInvocation( - ['git', 'rev-parse', 'HEAD'], - '/flutter', - ); - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'init', - '--instance', 'flutter', - '--work-dir', '/workDirectory/temp', - '--commit', '1234', - '--keys-file', '/workDirectory/keys.json', - '--failure-file', '/workDirectory/failures.json', - '--passfail', - '--crs', 'github', - '--patchset_id', '1234', - '--changelist', '49815', - '--cis', 'buildbucket', - '--jobid', '8885996262141582672', - ], - null, - ); - process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); - process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); - - // First call - await skiaClient.tryjobInit(); - - // Remove fake process result. - // If the init call is executed again, the fallback process will throw. - process.processResults.remove(goldctlInvocation); - - // Second call - await skiaClient.tryjobInit(); - }); - - test('throws for error state from imgtestAdd', () { - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'add', - '--work-dir', '/workDirectory/temp', - '--test-name', 'golden_file_test', - '--png-file', '/workDirectory/temp/golden_file_test.png', - '--passfail', - ], - null, - ); - process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); - - expect( - skiaClient.imgtestAdd('golden_file_test', goldenFile), - throwsException, - ); - }); - - test('correctly inits tryjob for luci', () async { - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB' : 'refs/pull/49815/head', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - final List ciArguments = skiaClient.getCIArguments(); - - expect( - ciArguments, - equals( - [ - '--changelist', '49815', - '--cis', 'buildbucket', - '--jobid', '8885996262141582672', - ], - ), - ); - }); - - test('Creates traceID correctly', () async { - String traceID; - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB' : 'refs/pull/49815/head', - }, - operatingSystem: 'linux' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - traceID = skiaClient.getTraceID('flutter.golden.1'); - expect( - traceID, - equals('ae18c7a6aa48e0685525dfe8fdf79003'), - ); - - // Browser - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', - 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', - 'GOLD_TRYJOB' : 'refs/pull/49815/head', - 'FLUTTER_TEST_BROWSER' : 'chrome', - }, - operatingSystem: 'linux' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - traceID = skiaClient.getTraceID('flutter.golden.1'); - expect( - traceID, - equals('e9d5c296c48e7126808520e9cc191243'), - ); - - // Locally - should defer to luci traceID - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - traceID = skiaClient.getTraceID('flutter.golden.1'); - expect( - traceID, - equals('9968695b9ae78cdb77cbb2be621ca2d6'), - ); - }); - - test('throws for error state from imgtestAdd', () { - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'add', - '--work-dir', '/workDirectory/temp', - '--test-name', 'golden_file_test', - '--png-file', '/workDirectory/temp/golden_file_test.png', - '--passfail', - ], - null, - ); - process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); - - expect( - skiaClient.imgtestAdd('golden_file_test', goldenFile), - throwsA( - isA().having((SkiaException error) => error.message, - 'message', - contains('result-state.json'), - ), - ), - ); - }); - - test('throws for error state from tryjobAdd', () { - final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') - ..createSync(recursive: true); - platform = FakePlatform( - environment: { - 'FLUTTER_ROOT': _kFlutterRoot, - 'GOLDCTL' : 'goldctl', - }, - operatingSystem: 'macos' - ); - - skiaClient = SkiaGoldClient( - workDirectory, - fs: fs, - process: process, - platform: platform, - httpClient: fakeHttpClient, - ); - - const RunInvocation goldctlInvocation = RunInvocation( - [ - 'goldctl', - 'imgtest', 'add', - '--work-dir', '/workDirectory/temp', - '--test-name', 'golden_file_test', - '--png-file', '/workDirectory/temp/golden_file_test.png', - '--passfail', - ], - null, - ); - process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); - process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); - - expect( - skiaClient.tryjobAdd('golden_file_test', goldenFile), - throwsA( - isA().having((SkiaException error) => error.message, - 'message', - contains('result-state.json'), - ), - ), - ); - }); - - group('Request Handling', () { - const String expectation = '55109a4bed52acc780530f7a9aeff6c0'; - - test('image bytes are processed properly', () async { - final Uri imageUrl = Uri.parse( - 'https://flutter-gold.skia.org/img/images/$expectation.png' - ); - final FakeHttpClientRequest fakeImageRequest = FakeHttpClientRequest(); - final FakeHttpImageResponse fakeImageResponse = FakeHttpImageResponse( - imageResponseTemplate() - ); - - fakeHttpClient.request = fakeImageRequest; - fakeImageRequest.response = fakeImageResponse; - - final List masterBytes = await skiaClient.getImageBytes(expectation); - - expect(fakeHttpClient.lastUri, imageUrl); - expect(masterBytes, equals(_kTestPngBytes)); - }); - }); - }); - group('FlutterGoldenFileComparator', () { late FlutterGoldenFileComparator comparator; @@ -650,7 +80,7 @@ void main() { namePrefix: namePrefix, ); await comparator.compare( - Uint8List.fromList(_kTestPngBytes), + Uint8List.fromList(kTestPngBytes), Uri.parse(fileName), ); expect(fakeSkiaClient.testNames.single, '$namePrefix.$libraryName.$fileName'); @@ -675,7 +105,7 @@ void main() { await expectLater( () async { return comparator.compare( - Uint8List.fromList(_kTestPngBytes), + Uint8List.fromList(kTestPngBytes), Uri.parse('flutter.golden_test.1'), ); }, @@ -694,12 +124,26 @@ void main() { test('calls init during compare', () { expect(fakeSkiaClient.initCalls, 0); comparator.compare( - Uint8List.fromList(_kTestPngBytes), + Uint8List.fromList(kTestPngBytes), Uri.parse('flutter.golden_test.1.png'), ); expect(fakeSkiaClient.initCalls, 1); }); + test('Passes on flaky flag to client, resets after comparing', () { + // Not flaky + expect(comparator.getAndResetFlakyMode(), isFalse); + comparator.enableFlakyMode(); + expect(fakeSkiaClient.calledWithFlaky, 0); + comparator.compare( + Uint8List.fromList(kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + expect(fakeSkiaClient.calledWithFlaky, 1); + // Flaky flag was reset during compare. + expect(comparator.getAndResetFlakyMode(), isFalse); + }); + test('does not call init in during construction', () { expect(fakeSkiaClient.initCalls, 0); FlutterPostSubmitFileComparator.fromDefaultComparator( @@ -793,7 +237,7 @@ void main() { await expectLater( () async { return comparator.compare( - Uint8List.fromList(_kTestPngBytes), + Uint8List.fromList(kTestPngBytes), Uri.parse('flutter.golden_test.1'), ); }, @@ -812,12 +256,27 @@ void main() { test('calls init during compare', () { expect(fakeSkiaClient.tryInitCalls, 0); comparator.compare( - Uint8List.fromList(_kTestPngBytes), + Uint8List.fromList(kTestPngBytes), Uri.parse('flutter.golden_test.1.png'), ); expect(fakeSkiaClient.tryInitCalls, 1); }); + test('Passes on flaky flag to client, resets after comparing', () { + // Not flaky + expect(comparator.getAndResetFlakyMode(), isFalse); + comparator.enableFlakyMode(); + expect(fakeSkiaClient.calledWithFlaky, 0); + comparator.compare( + Uint8List.fromList(kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + // Init & add were called with flaky set. + expect(fakeSkiaClient.calledWithFlaky, 1); + // Flaky flag was reset during compare. + expect(comparator.getAndResetFlakyMode(), isFalse); + }); + test('does not call init in during construction', () { expect(fakeSkiaClient.tryInitCalls, 0); FlutterPostSubmitFileComparator.fromDefaultComparator( @@ -907,6 +366,19 @@ void main() { }); group('Skipping', () { + test('Resets flaky flag after comparing', () { + // Not flaky + expect(comparator.getAndResetFlakyMode(), isFalse); + // Set flaky + comparator.enableFlakyMode(); + comparator.compare( + Uint8List.fromList(kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + // Flaky flag was reset during compare. + expect(comparator.getAndResetFlakyMode(), isFalse); + }); + group('correctly determines testing environment', () { test('returns true on Cirrus builds', () { platform = FakePlatform( @@ -971,7 +443,7 @@ void main() { const String hash = '55109a4bed52acc780530f7a9aeff6c0'; fakeSkiaClient.expectationForTestValues['flutter.golden_test.1'] = hash; - fakeSkiaClient.imageBytesValues[hash] =_kTestPngBytes; + fakeSkiaClient.imageBytesValues[hash] =kTestPngBytes; fakeSkiaClient.cleanTestNameValues['library.flutter.golden_test.1.png'] = 'flutter.golden_test.1'; }); @@ -979,7 +451,7 @@ void main() { await expectLater( () async { return comparator.compare( - Uint8List.fromList(_kTestPngBytes), + Uint8List.fromList(kTestPngBytes), Uri.parse('flutter.golden_test.1'), ); }, @@ -998,13 +470,25 @@ void main() { test('passes when bytes match', () async { expect( await comparator.compare( - Uint8List.fromList(_kTestPngBytes), + Uint8List.fromList(kTestPngBytes), Uri.parse('flutter.golden_test.1.png'), ), isTrue, ); }); + test('Passes when flaky', () { + // Not flaky + expect(comparator.getAndResetFlakyMode(), isFalse); + comparator.enableFlakyMode(); + comparator.compare( + Uint8List.fromList(kTestPngBytes), + Uri.parse('flutter.golden_test.1.png'), + ); + // Flaky flag was reset during compare. + expect(comparator.getAndResetFlakyMode(), isFalse); + }); + test('returns FlutterSkippingGoldenFileComparator when network connection is unavailable', () async { final FakeDirectory fakeDirectory = FakeDirectory(); fakeDirectory.existsSyncValue = true; @@ -1033,156 +517,3 @@ void main() { }); }); } - -@immutable -class RunInvocation { - const RunInvocation(this.command, this.workingDirectory); - - final List command; - final String? workingDirectory; - - @override - int get hashCode => Object.hash(Object.hashAll(command), workingDirectory); - - bool _commandEquals(List other) { - if (other == command) { - return true; - } - if (other.length != command.length) { - return false; - } - for (int index = 0; index < other.length; index += 1) { - if (other[index] != command[index]) { - return false; - } - } - return true; - } - - @override - bool operator ==(Object other) { - if (other.runtimeType != runtimeType) { - return false; - } - return other is RunInvocation - && _commandEquals(other.command) - && other.workingDirectory == workingDirectory; - } - - @override - String toString() => '$command ($workingDirectory)'; -} - -class FakeProcessManager extends Fake implements ProcessManager { - Map processResults = {}; - - /// Used if [processResults] does not contain a matching invocation. - ProcessResult? fallbackProcessResult; - - final List workingDirectories = []; - - @override - Future run( - List command, { - String? workingDirectory, - Map? environment, - bool includeParentEnvironment = true, - bool runInShell = false, - Encoding? stdoutEncoding = systemEncoding, - Encoding? stderrEncoding = systemEncoding, - }) async { - workingDirectories.add(workingDirectory); - final ProcessResult? result = processResults[RunInvocation(command.cast(), workingDirectory)]; - if (result == null && fallbackProcessResult == null) { - printOnFailure('ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.'); - fail('See above.'); - } - return result ?? fallbackProcessResult!; - } -} - -// See also dev/automated_tests/flutter_test/flutter_gold_test.dart -class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { - Map expectationForTestValues = {}; - Exception? getExpectationForTestThrowable; - @override - Future getExpectationForTest(String testName) async { - if (getExpectationForTestThrowable != null) { - throw getExpectationForTestThrowable!; - } - return expectationForTestValues[testName] ?? ''; - } - - @override - Future auth() async {} - - final List testNames = []; - - int initCalls = 0; - @override - Future imgtestInit() async => initCalls += 1; - @override - Future imgtestAdd(String testName, File goldenFile) async { - testNames.add(testName); - return true; - } - - int tryInitCalls = 0; - @override - Future tryjobInit() async => tryInitCalls += 1; - @override - Future tryjobAdd(String testName, File goldenFile) async => true; - - Map> imageBytesValues = >{}; - @override - Future> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!; - - Map cleanTestNameValues = {}; - @override - String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? ''; -} - -class FakeLocalFileComparator extends Fake implements LocalFileComparator { - @override - late Uri basedir; -} - -class FakeDirectory extends Fake implements Directory { - late bool existsSyncValue; - @override - bool existsSync() => existsSyncValue; - - @override - late Uri uri; -} - -class FakeHttpClient extends Fake implements HttpClient { - late Uri lastUri; - late FakeHttpClientRequest request; - - @override - Future getUrl(Uri url) async { - lastUri = url; - return request; - } -} - -class FakeHttpClientRequest extends Fake implements HttpClientRequest { - late FakeHttpImageResponse response; - - @override - Future close() async { - return response; - } -} - -class FakeHttpImageResponse extends Fake implements HttpClientResponse { - FakeHttpImageResponse(this.response); - - final List> response; - - @override - Future forEach(void Function(List element) action) async { - response.forEach(action); - } -} diff --git a/packages/flutter_goldens/test/skia_client_test.dart b/packages/flutter_goldens/test/skia_client_test.dart new file mode 100644 index 000000000000..f8a28fc62539 --- /dev/null +++ b/packages/flutter_goldens/test/skia_client_test.dart @@ -0,0 +1,716 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// See also dev/automated_tests/flutter_test/flutter_gold_test.dart + +import 'dart:convert'; +import 'dart:io' hide Directory; + +import 'package:file/file.dart'; +import 'package:file/memory.dart'; +import 'package:flutter_goldens/src/flutter_goldens_io.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:platform/platform.dart'; + +import 'utils/fakes.dart'; +import 'utils/json_templates.dart'; + +const String _kFlutterRoot = '/flutter'; + +void main() { + late SkiaGoldClient skiaClient; + late Directory workDirectory; + late MemoryFileSystem fs; + late FakePlatform platform; + late FakeProcessManager process; + late FakeHttpClient fakeHttpClient; + + setUp(() { + fs = MemoryFileSystem(); + platform = FakePlatform( + environment: {'FLUTTER_ROOT': _kFlutterRoot}, + operatingSystem: 'macos' + ); + process = FakeProcessManager(); + fakeHttpClient = FakeHttpClient(); + fs.directory(_kFlutterRoot).createSync(recursive: true); + }); + + setUp(() { + workDirectory = fs.directory('/workDirectory') + ..createSync(recursive: true); + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + }); + + test('web HTML test', () async { + platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'FLUTTER_ROOT': _kFlutterRoot, + 'FLUTTER_TEST_BROWSER': 'Chrome', + 'FLUTTER_WEB_RENDERER': 'html', + }, + operatingSystem: 'macos' + ); + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + '--add-test-optional-key', 'image_matching_algorithm:fuzzy', + '--add-test-optional-key', 'fuzzy_max_different_pixels:20', + '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:4', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); + final Map keys = { + 'Platform' : 'macos-browser', + 'CI' : 'luci', + 'markedFlaky' : 'false', + 'Browser' : 'Chrome', + 'WebRenderer' : 'html', + }; + + expect( + skiaClient.getKeysJSON(), + json.encode(keys), + ); + expect( + await skiaClient.imgtestAdd('golden_file_test.png', goldenFile), + isTrue, + ); + }); + + test('isFlaky sets right args - img test', () async { + platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'FLUTTER_ROOT': _kFlutterRoot, + 'FLUTTER_TEST_BROWSER': 'Chrome', + 'FLUTTER_WEB_RENDERER': 'html', + }, + operatingSystem: 'macos' + ); + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + '--add-test-optional-key', 'image_matching_algorithm:fuzzy', + '--add-test-optional-key', 'fuzzy_max_different_pixels:1000000000', + '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:1020', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); + final Map keys = { + 'Platform' : 'macos-browser', + 'CI' : 'luci', + 'markedFlaky' : 'false', + 'Browser' : 'Chrome', + 'WebRenderer' : 'html', + }; + + expect( + skiaClient.getKeysJSON(), + json.encode(keys), + ); + expect( + await skiaClient.imgtestAdd('golden_file_test.png', goldenFile, isFlaky: true), + isTrue, + ); + }); + + test('isFlaky sets right args - try job', () async { + platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'FLUTTER_ROOT': _kFlutterRoot, + 'FLUTTER_TEST_BROWSER': 'Chrome', + 'FLUTTER_WEB_RENDERER': 'html', + }, + operatingSystem: 'macos' + ); + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--add-test-optional-key', 'image_matching_algorithm:fuzzy', + '--add-test-optional-key', 'fuzzy_max_different_pixels:1000000000', + '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:1020', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); + final Map keys = { + 'Platform' : 'macos-browser', + 'CI' : 'luci', + 'markedFlaky' : 'false', + 'Browser' : 'Chrome', + 'WebRenderer' : 'html', + }; + + expect( + skiaClient.getKeysJSON(), + json.encode(keys), + ); + await skiaClient.tryjobAdd('golden_file_test.png', goldenFile, isFlaky: true); + }); + + test('web CanvasKit test', () async { + platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'FLUTTER_ROOT': _kFlutterRoot, + 'FLUTTER_TEST_BROWSER': 'Chrome', + 'FLUTTER_WEB_RENDERER': 'canvaskit', + }, + operatingSystem: 'macos' + ); + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); + final Map keys = { + 'Platform' : 'macos-browser', + 'CI' : 'luci', + 'markedFlaky' : 'false', + 'Browser' : 'Chrome', + 'WebRenderer' : 'canvaskit', + }; + + expect( + skiaClient.getKeysJSON(), + json.encode(keys), + ); + expect( + await skiaClient.imgtestAdd('golden_file_test.png', goldenFile), + isTrue, + ); + }); + + test('auth performs minimal work if already authorized', () async { + final File authFile = fs.file('/workDirectory/temp/auth_opt.json') + ..createSync(recursive: true); + authFile.writeAsStringSync(authTemplate()); + process.fallbackProcessResult = ProcessResult(123, 0, '', ''); + await skiaClient.auth(); + + expect(process.workingDirectories, isEmpty); + }); + + test('gsutil is checked when authorization file is present', () async { + final File authFile = fs.file('/workDirectory/temp/auth_opt.json') + ..createSync(recursive: true); + authFile.writeAsStringSync(authTemplate(gsutil: true)); + expect( + await skiaClient.clientIsAuthorized(), + isFalse, + ); + }); + + test('throws for error state from auth', () async { + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLD_SERVICE_ACCOUNT' : 'Service Account', + 'GOLDCTL' : 'goldctl', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect( + skiaClient.auth(), + throwsException, + ); + }); + + test('throws for error state from init', () { + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + const RunInvocation gitInvocation = RunInvocation( + ['git', 'rev-parse', 'HEAD'], + '/flutter', + ); + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'init', + '--instance', 'flutter', + '--work-dir', '/workDirectory/temp', + '--commit', '12345678', + '--keys-file', '/workDirectory/keys.json', + '--failure-file', '/workDirectory/failures.json', + '--passfail', + ], + null, + ); + process.processResults[gitInvocation] = ProcessResult(12345678, 0, '12345678', ''); + process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect( + skiaClient.imgtestInit(), + throwsException, + ); + }); + + test('Only calls init once', () async { + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + const RunInvocation gitInvocation = RunInvocation( + ['git', 'rev-parse', 'HEAD'], + '/flutter', + ); + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'init', + '--instance', 'flutter', + '--work-dir', '/workDirectory/temp', + '--commit', '1234', + '--keys-file', '/workDirectory/keys.json', + '--failure-file', '/workDirectory/failures.json', + '--passfail', + ], + null, + ); + process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); + process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + // First call + await skiaClient.imgtestInit(); + + // Remove fake process result. + // If the init call is executed again, the fallback process will throw. + process.processResults.remove(goldctlInvocation); + + // Second call + await skiaClient.imgtestInit(); + }); + + test('Only calls tryjob init once', () async { + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB' : 'refs/pull/49815/head', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + const RunInvocation gitInvocation = RunInvocation( + ['git', 'rev-parse', 'HEAD'], + '/flutter', + ); + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'init', + '--instance', 'flutter', + '--work-dir', '/workDirectory/temp', + '--commit', '1234', + '--keys-file', '/workDirectory/keys.json', + '--failure-file', '/workDirectory/failures.json', + '--passfail', + '--crs', 'github', + '--patchset_id', '1234', + '--changelist', '49815', + '--cis', 'buildbucket', + '--jobid', '8885996262141582672', + ], + null, + ); + process.processResults[gitInvocation] = ProcessResult(1234, 0, '1234', ''); + process.processResults[goldctlInvocation] = ProcessResult(5678, 0, '5678', ''); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + // First call + await skiaClient.tryjobInit(); + + // Remove fake process result. + // If the init call is executed again, the fallback process will throw. + process.processResults.remove(goldctlInvocation); + + // Second call + await skiaClient.tryjobInit(); + }); + + test('throws for error state from imgtestAdd', () { + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect( + skiaClient.imgtestAdd('golden_file_test', goldenFile), + throwsException, + ); + }); + + test('correctly inits tryjob for luci', () async { + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB' : 'refs/pull/49815/head', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + final List ciArguments = skiaClient.getCIArguments(); + + expect( + ciArguments, + equals( + [ + '--changelist', '49815', + '--cis', 'buildbucket', + '--jobid', '8885996262141582672', + ], + ), + ); + }); + + test('Creates traceID correctly', () async { + String traceID; + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB' : 'refs/pull/49815/head', + }, + operatingSystem: 'linux' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + traceID = skiaClient.getTraceID('flutter.golden.1'); + expect( + traceID, + equals('ae18c7a6aa48e0685525dfe8fdf79003'), + ); + + // Browser + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB' : 'refs/pull/49815/head', + 'FLUTTER_TEST_BROWSER' : 'chrome', + }, + operatingSystem: 'linux' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + traceID = skiaClient.getTraceID('flutter.golden.1'); + expect( + traceID, + equals('e9d5c296c48e7126808520e9cc191243'), + ); + + // Locally - should defer to luci traceID + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + traceID = skiaClient.getTraceID('flutter.golden.1'); + expect( + traceID, + equals('9968695b9ae78cdb77cbb2be621ca2d6'), + ); + }); + + test('throws for error state from imgtestAdd', () { + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect( + skiaClient.imgtestAdd('golden_file_test', goldenFile), + throwsA( + isA().having((SkiaException error) => error.message, + 'message', + contains('result-state.json'), + ), + ), + ); + }); + + test('throws for error state from tryjobAdd', () { + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 1, 'Expected failure', 'Expected failure'); + process.fallbackProcessResult = ProcessResult(123, 1, 'Fallback failure', 'Fallback failure'); + + expect( + skiaClient.tryjobAdd('golden_file_test', goldenFile), + throwsA( + isA().having((SkiaException error) => error.message, + 'message', + contains('result-state.json'), + ), + ), + ); + }); + + group('Request Handling', () { + const String expectation = '55109a4bed52acc780530f7a9aeff6c0'; + + test('image bytes are processed properly', () async { + final Uri imageUrl = Uri.parse( + 'https://flutter-gold.skia.org/img/images/$expectation.png' + ); + final FakeHttpClientRequest fakeImageRequest = FakeHttpClientRequest(); + final FakeHttpImageResponse fakeImageResponse = FakeHttpImageResponse( + imageResponseTemplate() + ); + + fakeHttpClient.request = fakeImageRequest; + fakeImageRequest.response = fakeImageResponse; + + final List masterBytes = await skiaClient.getImageBytes(expectation); + + expect(fakeHttpClient.lastUri, imageUrl); + expect(masterBytes, equals(kTestPngBytes)); + }); + }); +} diff --git a/packages/flutter_goldens/test/utils/fakes.dart b/packages/flutter_goldens/test/utils/fakes.dart new file mode 100644 index 000000000000..1ec29014dca5 --- /dev/null +++ b/packages/flutter_goldens/test/utils/fakes.dart @@ -0,0 +1,205 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// See also dev/automated_tests/flutter_test/flutter_gold_test.dart + +import 'dart:convert'; +import 'dart:io' hide Directory; + +import 'package:file/file.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_goldens/src/flaky_goldens.dart'; +import 'package:flutter_goldens/src/flutter_goldens_io.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:process/process.dart'; + +// 1x1 transparent pixel +const List kTestPngBytes = [ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, + 1, 0, 0, 0, 1, 8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 11, 73, 68, 65, 84, + 120, 1, 99, 97, 0, 2, 0, 0, 25, 0, 5, 144, 240, 54, 245, 0, 0, 0, 0, 73, 69, + 78, 68, 174, 66, 96, 130, +]; + +@immutable +class RunInvocation { + const RunInvocation(this.command, this.workingDirectory); + + final List command; + final String? workingDirectory; + + @override + int get hashCode => Object.hash(Object.hashAll(command), workingDirectory); + + bool _commandEquals(List other) { + if (other == command) { + return true; + } + if (other.length != command.length) { + return false; + } + for (int index = 0; index < other.length; index += 1) { + if (other[index] != command[index]) { + return false; + } + } + return true; + } + + @override + bool operator ==(Object other) { + if (other.runtimeType != runtimeType) { + return false; + } + return other is RunInvocation + && _commandEquals(other.command) + && other.workingDirectory == workingDirectory; + } + + @override + String toString() => '$command ($workingDirectory)'; +} + +class FakeProcessManager extends Fake implements ProcessManager { + Map processResults = {}; + + /// Used if [processResults] does not contain a matching invocation. + ProcessResult? fallbackProcessResult; + + final List workingDirectories = []; + + @override + Future run( + List command, { + String? workingDirectory, + Map? environment, + bool includeParentEnvironment = true, + bool runInShell = false, + Encoding? stdoutEncoding = systemEncoding, + Encoding? stderrEncoding = systemEncoding, + }) async { + workingDirectories.add(workingDirectory); + final ProcessResult? result = processResults[RunInvocation(command.cast(), workingDirectory)]; + if (result == null && fallbackProcessResult == null) { + printOnFailure('ProcessManager.run was called with $command ($workingDirectory) unexpectedly - $processResults.'); + fail('See above.'); + } + return result ?? fallbackProcessResult!; + } +} + +// See also dev/automated_tests/flutter_test/flutter_gold_test.dart +class FakeSkiaGoldClient extends Fake implements SkiaGoldClient { + Map expectationForTestValues = {}; + Exception? getExpectationForTestThrowable; + @override + Future getExpectationForTest(String testName) async { + if (getExpectationForTestThrowable != null) { + throw getExpectationForTestThrowable!; + } + return expectationForTestValues[testName] ?? ''; + } + + @override + Future auth() async {} + + final List testNames = []; + + int initCalls = 0; + int calledWithFlaky = 0; + @override + Future imgtestInit({ bool isFlaky = false }) async { + initCalls += 1; + if (isFlaky) { + calledWithFlaky += 1; + } + } + @override + Future imgtestAdd(String testName, File goldenFile, { bool isFlaky = false }) async { + testNames.add(testName); + if (isFlaky) { + calledWithFlaky += 1; + } + return true; + } + + int tryInitCalls = 0; + @override + Future tryjobInit({ bool isFlaky = false }) async { + tryInitCalls += 1; + if (isFlaky) { + calledWithFlaky += 1; + } + } + + @override + Future tryjobAdd(String testName, File goldenFile, { bool isFlaky = false }) async { + if (isFlaky) { + calledWithFlaky += 1; + } + return true; + } + + Map> imageBytesValues = >{}; + @override + Future> getImageBytes(String imageHash) async => imageBytesValues[imageHash]!; + + Map cleanTestNameValues = {}; + @override + String cleanTestName(String fileName) => cleanTestNameValues[fileName] ?? ''; +} + +class FakeFlakyLocalFileComparator extends FakeLocalFileComparator with FlakyGoldenMixin {} + +class FakeLocalFileComparator extends Fake implements LocalFileComparator { + @override + late Uri basedir; + + @override + Uri getTestUri(Uri key, int? version) => Uri.parse('fake'); + + @override + @override + Future compare(Uint8List imageBytes, Uri golden) async => true; +} + +class FakeDirectory extends Fake implements Directory { + late bool existsSyncValue; + @override + bool existsSync() => existsSyncValue; + + @override + late Uri uri; +} + +class FakeHttpClient extends Fake implements HttpClient { + late Uri lastUri; + late FakeHttpClientRequest request; + + @override + Future getUrl(Uri url) async { + lastUri = url; + return request; + } +} + +class FakeHttpClientRequest extends Fake implements HttpClientRequest { + late FakeHttpImageResponse response; + + @override + Future close() async { + return response; + } +} + +class FakeHttpImageResponse extends Fake implements HttpClientResponse { + FakeHttpImageResponse(this.response); + + final List> response; + + @override + Future forEach(void Function(List element) action) async { + response.forEach(action); + } +} diff --git a/packages/flutter_goldens/test/json_templates.dart b/packages/flutter_goldens/test/utils/json_templates.dart similarity index 100% rename from packages/flutter_goldens/test/json_templates.dart rename to packages/flutter_goldens/test/utils/json_templates.dart diff --git a/packages/flutter_goldens_client/lib/skia_client.dart b/packages/flutter_goldens_client/lib/skia_client.dart index b30b736bc339..3b81c67a136b 100644 --- a/packages/flutter_goldens_client/lib/skia_client.dart +++ b/packages/flutter_goldens_client/lib/skia_client.dart @@ -8,6 +8,7 @@ import 'dart:io' as io; import 'package:crypto/crypto.dart'; import 'package:file/file.dart'; import 'package:file/local.dart'; +import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import 'package:platform/platform.dart'; import 'package:process/process.dart'; @@ -138,7 +139,7 @@ class SkiaGoldClient { /// The `imgtest` command collects and uploads test results to the Skia Gold /// backend, the `init` argument initializes the current test. Used by the /// [FlutterPostSubmitFileComparator]. - Future imgtestInit() async { + Future imgtestInit({ bool isFlaky = false }) async { // This client has already been initialized if (_initialized) { return; @@ -147,7 +148,7 @@ class SkiaGoldClient { final File keys = workDirectory.childFile('keys.json'); final File failures = workDirectory.childFile('failures.json'); - await keys.writeAsString(_getKeysJSON()); + await keys.writeAsString(getKeysJSON(isFlaky: isFlaky)); await failures.create(); final String commitHash = await _getCurrentCommit(); @@ -199,7 +200,7 @@ class SkiaGoldClient { /// /// The [testName] and [goldenFile] parameters reference the current /// comparison being evaluated by the [FlutterPostSubmitFileComparator]. - Future imgtestAdd(String testName, File goldenFile) async { + Future imgtestAdd(String testName, File goldenFile, { bool isFlaky = false }) async { final List imgtestCommand = [ _goldctl, 'imgtest', 'add', @@ -209,7 +210,7 @@ class SkiaGoldClient { '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, '--passfail', - ..._getPixelMatchingArguments(), + ..._getPixelMatchingArguments(isFlaky: isFlaky), ]; final io.ProcessResult result = await process.run(imgtestCommand); @@ -259,7 +260,7 @@ class SkiaGoldClient { /// The `imgtest` command collects and uploads test results to the Skia Gold /// backend, the `init` argument initializes the current tryjob. Used by the /// [FlutterPreSubmitFileComparator]. - Future tryjobInit() async { + Future tryjobInit({ bool isFlaky = false }) async { // This client has already been initialized if (_tryjobInitialized) { return; @@ -268,7 +269,7 @@ class SkiaGoldClient { final File keys = workDirectory.childFile('keys.json'); final File failures = workDirectory.childFile('failures.json'); - await keys.writeAsString(_getKeysJSON()); + await keys.writeAsString(getKeysJSON(isFlaky: isFlaky)); await failures.create(); final String commitHash = await _getCurrentCommit(); @@ -323,7 +324,7 @@ class SkiaGoldClient { /// /// The [testName] and [goldenFile] parameters reference the current /// comparison being evaluated by the [FlutterPreSubmitFileComparator]. - Future tryjobAdd(String testName, File goldenFile) async { + Future tryjobAdd(String testName, File goldenFile, { bool isFlaky = false}) async { final List imgtestCommand = [ _goldctl, 'imgtest', 'add', @@ -332,7 +333,7 @@ class SkiaGoldClient { .path, '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, - ..._getPixelMatchingArguments(), + ..._getPixelMatchingArguments(isFlaky: isFlaky), ]; final io.ProcessResult result = await process.run(imgtestCommand); @@ -362,6 +363,42 @@ class SkiaGoldClient { } } + List _getPixelMatchingArguments({ required bool isFlaky }) { + if (isFlaky) { + return _getFlakyPixelMatchingArguments(); + } else { + return _getNormalPixelMatchingArguments(); + } + } + + List _getFlakyPixelMatchingArguments() { + // The algorithm to be used when matching images. The available options are: + // - "fuzzy": Allows for customizing the thresholds of pixel differences. + // - "sobel": Same as "fuzzy" but performs edge detection before performing + // a fuzzy match. + const String algorithm = 'fuzzy'; + + // The number of pixels in this image that are allowed to differ from the + // baseline. + // + // The chosen number - 1 billion - indicates that a flaky test should pass + // no matter how many pixels are different from the master golden. + const int maxDifferentPixels = 1000 * 1000 * 1000; + + // The maximum acceptable difference per pixel. + // + // The chosen number - 1020 - is the maximum supported pixel delta and + // indicates that a flaky test should pass no matter how far the new pixels + // deviate from the master golden. + const int pixelDeltaThreshold = 1020; + + return [ + '--add-test-optional-key', 'image_matching_algorithm:$algorithm', + '--add-test-optional-key', 'fuzzy_max_different_pixels:$maxDifferentPixels', + '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:$pixelDeltaThreshold', + ]; + } + // Constructs arguments for `goldctl` for controlling how pixels are compared. // // For AOT and CanvasKit exact pixel matching is used. For the HTML renderer @@ -369,7 +406,7 @@ class SkiaGoldClient { // because Chromium cannot exactly reproduce the same golden on all computers. // It seems to depend on the hardware/OS/driver combination. However, those // differences are very small (typically not noticeable to human eye). - List _getPixelMatchingArguments() { + List _getNormalPixelMatchingArguments() { // Only use fuzzy pixel matching in the HTML renderer. if (!_isBrowserTest || _isBrowserCanvasKitTest) { return const []; @@ -485,17 +522,17 @@ class SkiaGoldClient { /// Currently, the only key value pairs being tracked is the platform the /// image was rendered on, and for web tests, the browser the image was /// rendered on. - String _getKeysJSON() { + @visibleForTesting + String getKeysJSON({ bool isFlaky = false}) { final Map keys = { 'Platform' : platform.operatingSystem, 'CI' : 'luci', + 'markedFlaky' : isFlaky.toString(), }; if (_isBrowserTest) { keys['Browser'] = _browserKey; keys['Platform'] = '${keys['Platform']}-browser'; - if (_isBrowserCanvasKitTest) { - keys['WebRenderer'] = 'canvaskit'; - } + keys['WebRenderer'] = _isBrowserCanvasKitTest ? 'canvaskit' : 'html'; } return json.encode(keys); } diff --git a/packages/flutter_test/lib/src/_goldens_web.dart b/packages/flutter_test/lib/src/_goldens_web.dart index 2b9b2bed39a3..4e33d4e0388d 100644 --- a/packages/flutter_test/lib/src/_goldens_web.dart +++ b/packages/flutter_test/lib/src/_goldens_web.dart @@ -41,12 +41,12 @@ Future compareLists(List test, List master) async { /// * [matchesGoldenFile], the function from [flutter_test] that invokes the /// comparator. class DefaultWebGoldenComparator extends WebGoldenComparator { - /// Creates a new [DefaultWebGoldenComparator] for the specified [testFile]. + /// Creates a new [DefaultWebGoldenComparator] for the specified [testUri]. /// /// Golden file keys will be interpreted as file paths relative to the - /// directory in which [testFile] resides. + /// directory in which [testUri] resides. /// - /// The [testFile] URL must represent a file. + /// The [testUri] URL must represent a file. DefaultWebGoldenComparator(this.testUri); /// The test file currently being executed. diff --git a/packages/flutter_test/lib/src/goldens.dart b/packages/flutter_test/lib/src/goldens.dart index 8f0414511050..b289fbe5a404 100644 --- a/packages/flutter_test/lib/src/goldens.dart +++ b/packages/flutter_test/lib/src/goldens.dart @@ -238,6 +238,14 @@ set webGoldenComparator(WebGoldenComparator value) { _webGoldenComparator = value; } +/// The URI of the test file currently being executed. +/// +/// This variable is populated by the Flutter Tool automatically. +/// +/// Golden file keys will be interpreted as file paths relative to the directory +/// in which this file resides. +late Uri webTestUri; + /// Whether golden files should be automatically updated during tests rather /// than compared to the image bytes recorded by the tests. /// diff --git a/packages/flutter_tools/lib/src/test/flutter_web_goldens.dart b/packages/flutter_tools/lib/src/test/flutter_web_goldens.dart index fa0edeab2ffa..253f7653db8d 100644 --- a/packages/flutter_tools/lib/src/test/flutter_web_goldens.dart +++ b/packages/flutter_tools/lib/src/test/flutter_web_goldens.dart @@ -98,14 +98,14 @@ class TestGoldenComparator { return _processManager.start(command, environment: environment); } - Future compareGoldens(Uri testUri, Uint8List bytes, Uri goldenKey, bool? updateGoldens) async { + Future compareGoldens(Uri testUri, Uint8List bytes, Uri goldenKey, bool? updateGoldens, Map? customProperties) async { final File imageFile = await (await tempDir.createTemp('image')).childFile('image').writeAsBytes(bytes); final TestGoldenComparatorProcess? process = await _processForTestFile(testUri); if (process == null) { return 'process was null'; } - process.sendCommand(imageFile, goldenKey, updateGoldens); + process.sendCommand(imageFile, goldenKey, updateGoldens, customProperties); final Map result = await process.getResponse(); @@ -152,11 +152,13 @@ class TestGoldenComparatorProcess { await process.exitCode; } - void sendCommand(File imageFile, Uri? goldenKey, bool? updateGoldens) { + void sendCommand(File imageFile, Uri? goldenKey, bool? updateGoldens, Map? customProperties) { final Object command = jsonEncode({ 'imageFile': imageFile.path, 'key': goldenKey.toString(), 'update': updateGoldens, + if (customProperties != null) + 'customProperties': customProperties, }); _logger.printTrace('Preparing to send command: $command'); process.stdin.writeln(command); @@ -168,7 +170,22 @@ class TestGoldenComparatorProcess { return streamIterator.current; } + /// Generates the source code for the comparator process for the test file. + /// + /// If a test configuation exists for the tested package, uses its + /// implementation. Otherwise, uses the default implementation. static String generateBootstrap(File testFile, Uri testUri, {required Logger logger}) { + final File? webTestConfigFile = findWebTestConfigFile(testFile, logger); + if (webTestConfigFile != null) { + return _generateBootstrapWithWebTestConfig(webTestConfigFile, testFile, testUri); + } else { + return _generateBasicBootstrap(testFile, testUri, logger: logger); + } + } + + // Generates the bootstrap used by tests that either don't have a test + // configuration file, or don't have a `flutter_web_test_config.dart`. + static String _generateBasicBootstrap(File testFile, Uri testUri, {required Logger logger}) { final File? testConfigFile = findTestConfigFile(testFile, logger); // Generate comparator process for the file. return ''' @@ -211,6 +228,19 @@ void main() async { } } ${testConfigFile != null ? '});' : ''} +} + '''; + } + + // Generates the bootstrap used by tests that have a `flutter_web_test_config.dart`. + static String _generateBootstrapWithWebTestConfig(File webTestConfigFile, File testFile, Uri testUri) { + return ''' +import 'package:flutter_test/flutter_test.dart'; +import '${Uri.file(webTestConfigFile.path)}' as web_test_config; +void main() async { + final String testUri = '$testUri'; + goldenFileComparator = LocalFileComparator(Uri.parse(testUri)); + await web_test_config.startWebTestHostConfiguration(testUri); } '''; } diff --git a/packages/flutter_tools/lib/src/test/flutter_web_platform.dart b/packages/flutter_tools/lib/src/test/flutter_web_platform.dart index 61e4fe0b0a1f..8821f00c179f 100644 --- a/packages/flutter_tools/lib/src/test/flutter_web_platform.dart +++ b/packages/flutter_tools/lib/src/test/flutter_web_platform.dart @@ -343,10 +343,9 @@ class FlutterWebPlatform extends PlatformPlugin { } Future _goldenFileHandler(shelf.Request request) async { - if (request.url.path.contains('flutter_goldens')) { - final Map body = json.decode(await request.readAsString()) as Map; - final Uri goldenKey = Uri.parse(body['key']! as String); - final Uri testUri = Uri.parse(body['testUri']! as String); + if (request.method == 'POST' && request.url.path.contains('flutter_goldens')) { + final String requestJson = await request.readAsString(); + final Map body = json.decode(requestJson) as Map; final num width = body['width']! as num; final num height = body['height']! as num; Uint8List bytes; @@ -383,7 +382,10 @@ class FlutterWebPlatform extends PlatformPlugin { return shelf.Response.ok('Unknown error, bytes is null'); } - final String? errorMessage = await _testGoldenComparator.compareGoldens(testUri, bytes, goldenKey, updateGoldens); + final Uri goldenKey = Uri.parse(body['key']! as String); + final Uri testUri = Uri.parse(body['testUri']! as String); + final Map? customProperties = body['customProperties'] as Map?; + final String? errorMessage = await _testGoldenComparator.compareGoldens(testUri, bytes, goldenKey, updateGoldens, customProperties); return shelf.Response.ok(errorMessage ?? 'true'); } else { return shelf.Response.notFound('Not Found'); diff --git a/packages/flutter_tools/lib/src/test/test_config.dart b/packages/flutter_tools/lib/src/test/test_config.dart index 59298a469c14..8b460fb18557 100644 --- a/packages/flutter_tools/lib/src/test/test_config.dart +++ b/packages/flutter_tools/lib/src/test/test_config.dart @@ -9,23 +9,36 @@ import '../base/logger.dart'; /// test harness if it exists in the project directory hierarchy. const String _kTestConfigFileName = 'flutter_test_config.dart'; +/// The name of the web test configuration file that will be discovered by the +/// test harness if it exists in the project directory hierarchy. +const String _kWebTestConfigFileName = 'flutter_web_test_config.dart'; + /// The name of the file that signals the root of the project and that will /// cause the test harness to stop scanning for configuration files. const String _kProjectRootSentinel = 'pubspec.yaml'; /// Find the `flutter_test_config.dart` file for a specific test file. File? findTestConfigFile(File testFile, Logger logger) { + return _findConfigFile(testFile, _kTestConfigFileName, logger); +} + +/// Find the `flutter_web_test_config.dart` file for a specific test file. +File? findWebTestConfigFile(File testFile, Logger logger) { + return _findConfigFile(testFile, _kWebTestConfigFileName, logger); +} + +File? _findConfigFile(File testFile, String configFileName, Logger logger) { File? testConfigFile; Directory directory = testFile.parent; while (directory.path != directory.parent.path) { - final File configFile = directory.childFile(_kTestConfigFileName); + final File configFile = directory.childFile(configFileName); if (configFile.existsSync()) { - logger.printTrace('Discovered $_kTestConfigFileName in ${directory.path}'); + logger.printTrace('Discovered $configFileName in ${directory.path}'); testConfigFile = configFile; break; } if (directory.childFile(_kProjectRootSentinel).existsSync()) { - logger.printTrace('Stopping scan for $_kTestConfigFileName; ' + logger.printTrace('Stopping scan for $configFileName; ' 'found project root at ${directory.path}'); break; } diff --git a/packages/flutter_tools/lib/src/web/bootstrap.dart b/packages/flutter_tools/lib/src/web/bootstrap.dart index ccf714c99af1..0ae31a8ccd8a 100644 --- a/packages/flutter_tools/lib/src/web/bootstrap.dart +++ b/packages/flutter_tools/lib/src/web/bootstrap.dart @@ -224,7 +224,8 @@ String generateTestEntrypoint({ Future main() async { ui.debugEmulateFlutterTesterEnvironment = true; await ui.webOnlyInitializePlatform(); - webGoldenComparator = DefaultWebGoldenComparator(Uri.parse('${Uri.file(absolutePath)}')); + webTestUri = Uri.parse('${Uri.file(absolutePath)}'); + webGoldenComparator = DefaultWebGoldenComparator(webTestUri); (ui.window as dynamic).debugOverrideDevicePixelRatio(3.0); (ui.window as dynamic).webOnlyDebugPhysicalSizeOverride = const ui.Size(2400, 1800); diff --git a/packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart b/packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart index 3e653572b230..9166a58796d1 100644 --- a/packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart +++ b/packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart @@ -41,13 +41,13 @@ void main() { final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink; final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess, logger: BufferLogger.test()); - process.sendCommand(imageFile, goldenKey, false); + process.sendCommand(imageFile, goldenKey, false, {'additional data' : 'data'}); final Map response = await process.getResponse(); final String stringToStdin = ioSink.getAndClear(); expect(response, expectedResponse); - expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n'); + expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false,"customProperties":{"additional data":"data"}}\n'); }); testWithoutContext('can handle multiple requests', () async { @@ -64,18 +64,21 @@ void main() { final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink; final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess, logger: BufferLogger.test()); - process.sendCommand(imageFile, goldenKey, false); + process.sendCommand(imageFile, goldenKey, false, null); final Map response1 = await process.getResponse(); - process.sendCommand(imageFile2, goldenKey2, true); + process.sendCommand(imageFile2, goldenKey2, true, null); final Map response2 = await process.getResponse(); final String stringToStdin = ioSink.getAndClear(); expect(response1, expectedResponse1); expect(response2, expectedResponse2); - expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n{"imageFile":"second_test_image_file","key":"file://second_golden_key/","update":true}\n'); + expect( + stringToStdin, + '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n' + '{"imageFile":"second_test_image_file","key":"file://second_golden_key/","update":true}\n'); }); testWithoutContext('ignores anything that does not look like JSON', () async { @@ -94,7 +97,7 @@ Other JSON data after the initial data final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink; final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess,logger: BufferLogger.test()); - process.sendCommand(imageFile, goldenKey, false); + process.sendCommand(imageFile, goldenKey, false, null); final Map response = await process.getResponse(); final String stringToStdin = ioSink.getAndClear(); diff --git a/packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart b/packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart index 32ad2c2daab7..01e021ba70de 100644 --- a/packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart +++ b/packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart @@ -61,7 +61,7 @@ void main() { webRenderer: WebRendererMode.html, ); - final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null); expect(result, null); }); @@ -90,7 +90,7 @@ void main() { webRenderer: WebRendererMode.canvaskit, ); - final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null); expect(result, 'some message'); }); @@ -123,10 +123,10 @@ void main() { webRenderer: WebRendererMode.html, ); - final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null); expect(result1, 'some message'); - final String? result2 = await comparator.compareGoldens(testUri, imageBytes, goldenKey2, false); + final String? result2 = await comparator.compareGoldens(testUri, imageBytes, goldenKey2, false, null); expect(result2, 'some other message'); }); @@ -168,10 +168,10 @@ void main() { webRenderer: WebRendererMode.canvaskit, ); - final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + final String? result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null); expect(result1, 'some message'); - final String? result2 = await comparator.compareGoldens(testUri2, imageBytes, goldenKey2, false); + final String? result2 = await comparator.compareGoldens(testUri2, imageBytes, goldenKey2, false, null); expect(result2, 'some other message'); }); @@ -203,7 +203,7 @@ void main() { webRenderer: WebRendererMode.html, ); - final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + final String? result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false, null); expect(result, null); await comparator.close(); diff --git a/packages/flutter_tools/test/integration.shard/forbidden_imports_test.dart b/packages/flutter_tools/test/integration.shard/forbidden_imports_test.dart index 92a886319a99..2981770a4026 100644 --- a/packages/flutter_tools/test/integration.shard/forbidden_imports_test.dart +++ b/packages/flutter_tools/test/integration.shard/forbidden_imports_test.dart @@ -117,7 +117,7 @@ void main() { final List allowedPaths = [ fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_platform.dart'), fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_web_platform.dart'), - fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'test_wrapper.dart'), + fileSystem.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_goldens.dart'), ]; bool isNotAllowed(FileSystemEntity entity) => allowedPaths.every((String path) => path != entity.path);