From 0b5e7c33b6097bc44b57d284ad0b051c4a476fc1 Mon Sep 17 00:00:00 2001 From: Angelo Silvestre Date: Wed, 2 Aug 2023 21:09:27 -0300 Subject: [PATCH] [SuperEditor][SuperTextLayout] Move golden runner to new package and fix local golden failures (Resolves #1265) (#1267) --- .dockerignore | 9 + golden_runner/.gitignore | 30 ++ golden_runner/.metadata | 10 + golden_runner/CHANGELOG.md | 3 + golden_runner/LICENSE | 1 + golden_runner/README.md | 41 +++ golden_runner/analysis_options.yaml | 4 + golden_runner/bin/goldens.dart | 17 + golden_runner/lib/golden_runner.dart | 3 + golden_runner/lib/src/commands.dart | 329 ++++++++++++++++++ golden_runner/pubspec.yaml | 17 + golden_tester.Dockerfile | 24 ++ super_editor/README_TESTS.md | 22 +- super_editor/golden_tester.Dockerfile | 13 - super_editor/pubspec.yaml | 11 +- super_editor/test/test_tools.dart | 1 + .../components/_components_test_utils.dart | 4 +- .../editor/components/paragraph_test.dart | 3 +- .../editor/mobile/mobile_selection_test.dart | 5 +- .../editor/supereditor_text_layout_test.dart | 25 +- .../super_textfield_scroll_test.dart | 6 +- .../super_textfield_text_alignment_test.dart | 13 +- .../super_textfield_toolbar_test.dart | 16 +- .../test_goldens/test_tools_goldens.dart | 232 ++++++++++++ super_editor/tool/goldens.dart | 229 ------------ super_text_layout/README_TESTS.md | 22 +- super_text_layout/golden_tester.Dockerfile | 13 - super_text_layout/pubspec.yaml | 3 + .../test_goldens/caret_layer_test.dart | 9 +- .../test_goldens/super_text_layers_test.dart | 9 +- .../test_goldens/super_text_test.dart | 6 +- .../test_goldens/test_tools_goldens.dart | 94 +++++ .../text_selection_layer_test.dart | 9 +- super_text_layout/tool/goldens.dart | 229 ------------ 34 files changed, 917 insertions(+), 545 deletions(-) create mode 100644 .dockerignore create mode 100644 golden_runner/.gitignore create mode 100644 golden_runner/.metadata create mode 100644 golden_runner/CHANGELOG.md create mode 100644 golden_runner/LICENSE create mode 100644 golden_runner/README.md create mode 100644 golden_runner/analysis_options.yaml create mode 100644 golden_runner/bin/goldens.dart create mode 100644 golden_runner/lib/golden_runner.dart create mode 100644 golden_runner/lib/src/commands.dart create mode 100644 golden_runner/pubspec.yaml create mode 100644 golden_tester.Dockerfile delete mode 100644 super_editor/golden_tester.Dockerfile create mode 100644 super_editor/test_goldens/test_tools_goldens.dart delete mode 100644 super_editor/tool/goldens.dart delete mode 100644 super_text_layout/golden_tester.Dockerfile create mode 100644 super_text_layout/test_goldens/test_tools_goldens.dart delete mode 100644 super_text_layout/tool/goldens.dart diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..fd1d202080 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +*/example/ +*/build/ +*/.dart_tool/ + +# We dont need git history inside the image. +.git + +# Ignore the golden failure directories because they will be mapped. +**/failures/ \ No newline at end of file diff --git a/golden_runner/.gitignore b/golden_runner/.gitignore new file mode 100644 index 0000000000..96486fd930 --- /dev/null +++ b/golden_runner/.gitignore @@ -0,0 +1,30 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.buildlog/ +.history +.svn/ +migrate_working_dir/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock. +/pubspec.lock +**/doc/api/ +.dart_tool/ +.packages +build/ diff --git a/golden_runner/.metadata b/golden_runner/.metadata new file mode 100644 index 0000000000..2b69774845 --- /dev/null +++ b/golden_runner/.metadata @@ -0,0 +1,10 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: "f8afcd5aa01b3ae6c55cb6e4c9fa4171e27a92f6" + channel: "master" + +project_type: package diff --git a/golden_runner/CHANGELOG.md b/golden_runner/CHANGELOG.md new file mode 100644 index 0000000000..41cc7d8192 --- /dev/null +++ b/golden_runner/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +* TODO: Describe initial release. diff --git a/golden_runner/LICENSE b/golden_runner/LICENSE new file mode 100644 index 0000000000..ba75c69f7f --- /dev/null +++ b/golden_runner/LICENSE @@ -0,0 +1 @@ +TODO: Add your license here. diff --git a/golden_runner/README.md b/golden_runner/README.md new file mode 100644 index 0000000000..13e9cf4570 --- /dev/null +++ b/golden_runner/README.md @@ -0,0 +1,41 @@ +This package contains a tool to run golden tests and update golden files in a docker container. + +The command should be run from the root of the package being tested. + +## Activate the package: + +```console +dart pub global activate --source path ./golden_runner +``` + +## Run golden tests: + +``` +# run all tests +flutter pub run ../golden_runner/tool/goldens test + +# run a single test +flutter pub run ../golden_runner/tool/goldens test --plain-name "something" + +# run all tests in a directory +flutter pub run ../golden_runner/tool/goldens test test_goldens/my_dir + +# run a single test in a directory +flutter pub run ../golden_runner/tool/goldens test --plain-name "something" test_goldens/my_dir +``` + +## Update golden files: + +``` +# update all goldens +flutter pub run ../golden_runner/tool/goldens update + +# update all goldens in a directory +flutter pub run ../golden_runner/tool/goldens update test_goldens/my_dir + +# update a single golden +flutter pub run ../golden_runner/tool/goldens update --plain-name "something" + +# update a single golden in a directory +flutter pub run ../golden_runner/tool/goldens update --plain-name "something" test_goldens/my_dir +``` diff --git a/golden_runner/analysis_options.yaml b/golden_runner/analysis_options.yaml new file mode 100644 index 0000000000..a5744c1cfb --- /dev/null +++ b/golden_runner/analysis_options.yaml @@ -0,0 +1,4 @@ +include: package:flutter_lints/flutter.yaml + +# Additional information about this file can be found at +# https://dart.dev/guides/language/analysis-options diff --git a/golden_runner/bin/goldens.dart b/golden_runner/bin/goldens.dart new file mode 100644 index 0000000000..32a696c56d --- /dev/null +++ b/golden_runner/bin/goldens.dart @@ -0,0 +1,17 @@ +import 'dart:io'; + +// ignore: depend_on_referenced_packages +import 'package:args/command_runner.dart'; +import 'package:golden_runner/golden_runner.dart'; + +Future main(List arguments) async { + final runner = CommandRunner("goldens", "A tool to run and update golden tests using docker") + ..addCommand(GoldenTestCommand()) + ..addCommand(UpdateGoldensCommand()); + + try { + await runner.run(arguments); + } on UsageException catch (e) { + stdout.write(e); + } +} diff --git a/golden_runner/lib/golden_runner.dart b/golden_runner/lib/golden_runner.dart new file mode 100644 index 0000000000..0fc9aef473 --- /dev/null +++ b/golden_runner/lib/golden_runner.dart @@ -0,0 +1,3 @@ +library golden_runner; + +export 'src/commands.dart'; diff --git a/golden_runner/lib/src/commands.dart b/golden_runner/lib/src/commands.dart new file mode 100644 index 0000000000..01301b6d36 --- /dev/null +++ b/golden_runner/lib/src/commands.dart @@ -0,0 +1,329 @@ +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:path/path.dart' as path; + +/// A [Command] which runs golden tests. +/// +/// The command run the tests in a Linux Docker container. It expects to be running in a "golden_tester" directory. +/// +/// Usage: `flutter pub run test ` +/// +/// Options: +/// +/// `--plain-name "test-name"`: Runs only the tests containing the given value in the test name. +/// +/// The target can be a directory or a file. This argument is optional. +/// +/// This is intended to be added as an [CommandRunner] command. +class GoldenTestCommand extends Command { + GoldenTestCommand() { + argParser.addOption( + 'plain-name', + help: 'A plain-text substring of the names of tests to run', + ); + } + + @override + String get name => 'test'; + + @override + String get description => 'Runs golden tests'; + + @override + Future run() async { + final args = argResults!; + + // The tool must run from the root of the package being tested. + // For example, /super_editor/super_text_layout. + // We take the last part of the directory as the package directory. + final packageDirectory = path.split(Directory.current.path).last; + + // Builds the image used to run the container. + // We can build the image even if it already exists. + // Docker will cache each step used in the Dockerfile, so subsequent builds will be faster. + await _buildDockerImage(); + + // Arguments that are placed after 'flutter test test_goldens'. + final cmdArguments = []; + + final name = args['plain-name']; + if (name is String) { + cmdArguments + ..add('--plain-name') + ..add(name); + } + + stdout.writeln('Running golden tests'); + + // Other arguments passed at the end of the command. + // For example, the test directory. + final rest = [...args.rest]; + + late String testDirOrTestFileName; + late String testBaseDirectory; + + if (rest.isNotEmpty) { + // An argument was passed after the command options. + // For example, in "flutter pub run tool/goldens test my_test_dir", "my_test_dir" is the first argument + // after the command. + // Use the first argument after the command options as the test directory or test file name + // and remove it from the rest. + testDirOrTestFileName = rest.removeAt(0); + + if (path.extension(testDirOrTestFileName).isNotEmpty) { + // A test file was given. + // Extract the directory name so we can list the sub-directories. + testBaseDirectory = path.dirname(testDirOrTestFileName); + } else { + // A test directory was given. + // Don't try to extract the directory name because it can return an empty string if it's the root directory. + // For example, passing "test_goldens" to dirname would return "". + testBaseDirectory = testDirOrTestFileName; + } + } else { + testDirOrTestFileName = 'test_goldens'; + testBaseDirectory = testDirOrTestFileName; + } + + final dirs = _findAllTestDirectories(testBaseDirectory); + + final volumeMappings = _generateFailureDirectoriesMappings(packageDirectory, dirs); + + // Runs the container. + // + // --rm: Removes the container when it exits. + // + // --workdir: Sets the process working directory to the given package directory in the container. + await _runProcess( + executable: 'docker', + arguments: [ + 'run', + '--rm', + ...volumeMappings, + '--workdir', + '/golden_tester/$packageDirectory', + 'supereditor_golden_tester', + 'flutter', + 'test', + testDirOrTestFileName, + ...rest, + ...cmdArguments, + ], + description: 'Golden tests', + ); + + // Mapping the failure directories causes them to be created automatically, even without any failing test. + // Remove all the empty failure directories. + for (final dirName in dirs) { + final dir = Directory('$dirName/failures'); + if (dir.existsSync() && dir.listSync().isEmpty) { + dir.deleteSync(); + } + } + } + + /// Returns a list of docker command line arguments to configure the volume mappings for the test failure directories. + /// + /// [testDirectories] must be a list of relative paths to the working directory. + /// + /// This mappings are used so when a failure happens, the failure images are save in the host OS. + List _generateFailureDirectoriesMappings(String packageDirectory, List testDirectories) { + final mappings = []; + + for (final dir in testDirectories) { + mappings.add('-v'); + mappings.add('${Directory.current.path}/$dir/failures:/golden_tester/$packageDirectory/$dir/failures'); + } + + return mappings; + } + + /// Returns a list of sub-directories inside a root test directory as relative paths to the working directory. + /// + /// For example, "test_goldens/editor", "test_goldens/components". + /// + /// Ignores "failures" directories. + List _findAllTestDirectories(String rootTestDir) { + final dir = Directory(rootTestDir); + final subDirs = dir + .listSync(recursive: true) // + .whereType() + // Ensure we use linux path separator. + // The tool can run in a host OS which uses a different path separator. + // Without this, the volume ma + .map((e) => e.path.replaceAll(path.separator, '/')) + .where((e) => !e.endsWith('failures')) + .toList(); + return [rootTestDir, ...subDirs]; + } +} + +/// A [Command] which updates golden files. +/// +/// The command run the tests in a Linux Docker container. It expects to be running in a "golden_tester" directory. +/// +/// Usage: `flutter pub run update ` +/// +/// Options: +/// +/// `--plain-name "test-name"`: Update only the tests containing the given value in the test name. +/// +/// The target can be a directory or a file. This argument is optional. +/// +/// This is intended to be added as an [CommandRunner] command. +class UpdateGoldensCommand extends Command { + UpdateGoldensCommand() { + argParser.addOption( + 'plain-name', + help: 'A plain-text substring of the names of tests to run', + ); + } + + @override + String get description => 'Updates golden files'; + + @override + String get name => 'update'; + + @override + Future run() async { + final args = argResults!; + + // The tool must run from the root of the package being tested. + // For example, /super_editor/super_text_layout. + // We take the last part of the directory as the package directory. + final packageDirectory = path.split(Directory.current.path).last; + + // Builds the image used to run the container. + // We can build the image even if it already exists. + // Docker will cache each step used in the Dockerfile, so subsequent builds will be faster. + await _buildDockerImage(); + + // Arguments that are placed after 'flutter test test_goldens'. + final cmdArguments = []; + + final name = args['plain-name']; + if (name is String) { + cmdArguments + ..add('--plain-name') + ..add(name); + } + + stdout.writeln('Updating golden files'); + + // Other arguments passed at the end of the command. + // For example, the test directory. + final rest = [...args.rest]; + + late String testDirOrTestFileName; + late String testBaseDirectory; + + if (rest.isNotEmpty) { + // An argument was passed after the command options. + // For example, in "flutter pub run tool/goldens test my_test_dir", "my_test_dir" is the first argument + // after the command. + // Use the first argument after the command options as the test directory or test file name + // and remove it from the rest. + testDirOrTestFileName = rest.removeAt(0); + + if (path.extension(testDirOrTestFileName).isNotEmpty) { + // A test file was given. + // Extract the directory name so we can list the sub-directories. + testBaseDirectory = path.dirname(testDirOrTestFileName); + } else { + // A test directory was given. + // Don't try to extract the directory name because it can return an empty string if it's the root directory. + // For example, passing "test_goldens" to dirname would return "". + testBaseDirectory = testDirOrTestFileName; + } + } else { + testDirOrTestFileName = 'test_goldens'; + testBaseDirectory = testDirOrTestFileName; + } + + // Runs the container. + // + // --rm: Removes the container when it exits. + // + // -v: Mounts the directory containing the tests of the host machine into the container. + // This is used to write the new golden files directly on the host OS. + // + // --workdir: Sets the working directory to /super_editor/super_text_layout in the container. + await _runProcess( + executable: 'docker', + arguments: [ + 'run', + '--rm', + '-v', + '${Directory.current.path}/$testBaseDirectory:/golden_tester/$packageDirectory/$testBaseDirectory', + '--workdir', + '/golden_tester/$packageDirectory', + 'supereditor_golden_tester', + 'flutter', + 'test', + '--update-goldens', + testDirOrTestFileName, + ...rest, + ...cmdArguments, + ], + description: 'Update goldens', + ); + } +} + +/// Builds a linux docker image to run the tests. +/// +/// The golden_tester.Dockerfile is used to build this image. +Future _buildDockerImage() async { + stdout.write('building image'); + + await _runProcess( + executable: 'docker', + arguments: [ + 'build', + '-f', + './golden_tester.Dockerfile', + '-t', + 'supereditor_golden_tester', + '.', + ], + // We need to use the repository root as the working directory to be able to copy all of the files + // in this repository, not just the package directory. + workingDirectory: '../', + description: 'Image build', + ); +} + +/// Runs [executable] with the given [arguments]. +/// +/// [executable] could be an absolute path or it could be resolved from the PATH. +/// +/// The [arguments] must contain any modifiers, like `-`, `--` or `/`. +/// +/// Use [workingDirectory] to set the working directory for the process. +/// +/// The child process stdout and stderr are written to the current process stdout. +/// +/// Throws and exception using [description] in the message if the process exists with a non-zero exit code. +Future _runProcess({ + required String executable, + required List arguments, + required String description, + String? workingDirectory, +}) async { + final process = await Process.start( + executable, + arguments, + workingDirectory: workingDirectory, + ); + + stdout.addStream(process.stdout); + stderr.addStream(process.stderr); + + final exitCode = await process.exitCode; + + if (exitCode != 0) { + throw Exception('$description failed'); + } +} diff --git a/golden_runner/pubspec.yaml b/golden_runner/pubspec.yaml new file mode 100644 index 0000000000..2f03de03be --- /dev/null +++ b/golden_runner/pubspec.yaml @@ -0,0 +1,17 @@ +name: golden_runner +description: Commands to test and update goldens in a Docker container +version: 0.0.1 +homepage: + +executables: + goldens: + +environment: + sdk: ">=3.0.0 <4.0.0" + +dependencies: + args: ^2.3.1 + meta: ^1.8.0 + path: ^1.8.3 + +flutter: diff --git a/golden_tester.Dockerfile b/golden_tester.Dockerfile new file mode 100644 index 0000000000..b61f350cbd --- /dev/null +++ b/golden_tester.Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:latest + +ENV FLUTTER_HOME=${HOME}/sdks/flutter +ENV PATH ${PATH}:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin + +USER root + +RUN apt update + +RUN apt install -y git curl unzip + +# Print the Ubuntu version. Useful when there are failing tests. +RUN cat /etc/lsb-release + +# Invalidate the cache when flutter pushes a new commit. +ADD https://api.github.com/repos/flutter/flutter/git/refs/heads/master ./flutter-latest-master + +RUN git clone https://github.com/flutter/flutter.git ${FLUTTER_HOME} + +RUN flutter doctor + +# Copy the whole repo. +# We need this because we use local dependencies. +COPY ./ /golden_tester diff --git a/super_editor/README_TESTS.md b/super_editor/README_TESTS.md index c28f03f13e..02345334e2 100644 --- a/super_editor/README_TESTS.md +++ b/super_editor/README_TESTS.md @@ -1,35 +1,39 @@ # Running tests -In order to run the golden tests, docker must be installed. +In order to run the golden tests, docker must be installed. Activate the golden_runner with: + +```console +dart pub global activate --source path ../golden_runner +``` ## Run golden tests: ``` # run all tests -flutter pub run tool/goldens test +goldens test # run a single test -flutter pub run tool/goldens test --plain-name "something" +goldens test --plain-name "something" # run all tests in a directory -flutter pub run tool/goldens test test my_dir +goldens test test_goldens/my_dir # run a single test in a directory -flutter pub run tool/goldens test --plain-name "something" my_dir +goldens test --plain-name "something" test_goldens/my_dir ``` ## Update golden files: ``` # update all goldens -flutter pub run tool/goldens update +goldens update # update all goldens in a directory -flutter pub run tool/goldens update my_dir +goldens update test_goldens/my_dir # update a single golden -flutter pub run tool/goldens update --plain-name "something" +goldens update --plain-name "something" # update a single golden in a directory -flutter pub run tool/goldens update --plain-name "something" my_dir +goldens update --plain-name "something" test_goldens/my_dir ``` diff --git a/super_editor/golden_tester.Dockerfile b/super_editor/golden_tester.Dockerfile deleted file mode 100644 index 21dc39ab4f..0000000000 --- a/super_editor/golden_tester.Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:latest - -ENV FLUTTER_HOME=${HOME}/sdks/flutter -ENV PATH ${PATH}:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin - -USER root - -RUN apt update - -RUN apt install -y git curl unzip -RUN git clone https://github.com/flutter/flutter.git ${FLUTTER_HOME} - -RUN flutter doctor \ No newline at end of file diff --git a/super_editor/pubspec.yaml b/super_editor/pubspec.yaml index eccc83f0a4..a2c0b5b1cc 100644 --- a/super_editor/pubspec.yaml +++ b/super_editor/pubspec.yaml @@ -38,10 +38,10 @@ dependencies: flutter_test_robots: ^0.0.17 dependency_overrides: -# # Override to local mono-repo path so devs can test this repo -# # against changes that they're making to other mono-repo packages -# attributed_text: -# path: ../attributed_text + # # Override to local mono-repo path so devs can test this repo + # # against changes that they're making to other mono-repo packages + # attributed_text: + # path: ../attributed_text super_text_layout: path: ../super_text_layout # @@ -58,6 +58,9 @@ dev_dependencies: text_table: ^4.0.1 meta: ^1.8.0 args: ^2.3.1 + path: ^1.8.3 + golden_runner: + path: ../golden_runner flutter: # no Flutter configuration diff --git a/super_editor/test/test_tools.dart b/super_editor/test/test_tools.dart index 082921c8b8..f778bf5981 100644 --- a/super_editor/test/test_tools.dart +++ b/super_editor/test/test_tools.dart @@ -1,6 +1,7 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:logging/logging.dart'; import 'package:logging/logging.dart' as logging; import 'package:meta/meta.dart'; diff --git a/super_editor/test_goldens/editor/components/_components_test_utils.dart b/super_editor/test_goldens/editor/components/_components_test_utils.dart index 0ebb07fbfa..bdb93a6c1c 100644 --- a/super_editor/test_goldens/editor/components/_components_test_utils.dart +++ b/super_editor/test_goldens/editor/components/_components_test_utils.dart @@ -1,8 +1,10 @@ import 'package:flutter/material.dart'; import 'package:golden_toolkit/golden_toolkit.dart'; +import '../../test_tools_goldens.dart'; + void testComponentGolden(String description, Widget componentBuilder, String fileName) { - testGoldens(description, (tester) async { + testGoldensOnAndroid(description, (tester) async { tester.view ..physicalSize = const Size(600, 400) ..devicePixelRatio = 1.0; diff --git a/super_editor/test_goldens/editor/components/paragraph_test.dart b/super_editor/test_goldens/editor/components/paragraph_test.dart index 8dc11effd0..1df00d194c 100644 --- a/super_editor/test_goldens/editor/components/paragraph_test.dart +++ b/super_editor/test_goldens/editor/components/paragraph_test.dart @@ -3,10 +3,11 @@ import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_editor/super_editor.dart'; import '../../../test/super_editor/document_test_tools.dart'; +import '../../test_tools_goldens.dart'; void main() { group('SuperEditor', () { - testGoldens('displays paragraphs with different alignments', (tester) async { + testGoldensOnAndroid('displays paragraphs with different alignments', (tester) async { await tester.createDocument().withCustomContent(_createParagraphTestDoc()).pump(); await screenMatchesGolden(tester, 'paragraph_alignments'); diff --git a/super_editor/test_goldens/editor/mobile/mobile_selection_test.dart b/super_editor/test_goldens/editor/mobile/mobile_selection_test.dart index da8febc98e..7a5aeea825 100644 --- a/super_editor/test_goldens/editor/mobile/mobile_selection_test.dart +++ b/super_editor/test_goldens/editor/mobile/mobile_selection_test.dart @@ -6,6 +6,9 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_editor/super_editor.dart'; +import '../../../test/test_tools.dart'; +import '../../test_tools_goldens.dart'; + void main() { group('SuperEditor', () { group('mobile selection', () { @@ -722,7 +725,7 @@ void _testParagraphSelection( ) { final docKey = GlobalKey(); - testGoldens(description, (tester) async { + testGoldensOnAndroid(description, (tester) async { tester.view ..physicalSize = const Size(800, 200) ..devicePixelRatio = 1.0; diff --git a/super_editor/test_goldens/editor/supereditor_text_layout_test.dart b/super_editor/test_goldens/editor/supereditor_text_layout_test.dart index 6c31ff629e..c3e32887ce 100644 --- a/super_editor/test_goldens/editor/supereditor_text_layout_test.dart +++ b/super_editor/test_goldens/editor/supereditor_text_layout_test.dart @@ -6,11 +6,12 @@ import 'package:super_editor/src/test/super_editor_test/supereditor_robot.dart'; import 'package:super_editor/super_editor.dart'; import '../../test/super_editor/document_test_tools.dart'; +import '../test_tools_goldens.dart'; void main() { group('SuperEditor', () { group('applies textScaleFactor', () { - testGoldens('for paragraph', (tester) async { + testGoldensOnAndroid('for paragraph', (tester) async { await _buildTextScaleScaffold( tester, regularEditor: _buildSuperEditorFromMarkdown( @@ -26,7 +27,7 @@ void main() { await screenMatchesGolden(tester, 'text-scaling-paragraph'); }); - testGoldens('for paragraph with collapsed selection', (tester) async { + testGoldensOnAndroid('for paragraph with collapsed selection', (tester) async { final regularEditorKey = GlobalKey(); final scaledEditorKey = GlobalKey(); @@ -61,7 +62,7 @@ void main() { await screenMatchesGolden(tester, 'text-scaling-paragraph-collapsed-selection'); }); - testGoldens('for paragraph with expanded selection', (tester) async { + testGoldensOnAndroid('for paragraph with expanded selection', (tester) async { final regularEditorKey = GlobalKey(); final scaledEditorKey = GlobalKey(); @@ -96,7 +97,7 @@ void main() { await screenMatchesGolden(tester, 'text-scaling-paragraph-expanded-selection'); }); - testGoldens('for unordered list item', (tester) async { + testGoldensOnAndroid('for unordered list item', (tester) async { await _buildTextScaleScaffold( tester, regularEditor: _buildSuperEditorFromMarkdown( @@ -112,7 +113,7 @@ void main() { await screenMatchesGolden(tester, 'text-scaling-unordered-list'); }); - testGoldens('for ordered list item', (tester) async { + testGoldensOnAndroid('for ordered list item', (tester) async { await _buildTextScaleScaffold( tester, regularEditor: _buildSuperEditorFromMarkdown( @@ -128,7 +129,7 @@ void main() { await screenMatchesGolden(tester, 'text-scaling-ordered-list'); }); - testGoldens('for header', (tester) async { + testGoldensOnAndroid('for header', (tester) async { await _buildTextScaleScaffold( tester, regularEditor: _buildSuperEditorFromMarkdown( @@ -141,10 +142,13 @@ void main() { ), ); - await screenMatchesGolden(tester, 'text-scaling-header'); + await expectLater( + find.byType(MaterialApp).first, + matchesGoldenFileWithPixelAllowance("goldens/text-scaling-header.png", 0), + ); }); - testGoldens('for blockquote', (tester) async { + testGoldensOnAndroid('for blockquote', (tester) async { await _buildTextScaleScaffold( tester, regularEditor: _buildSuperEditorFromMarkdown( @@ -157,7 +161,10 @@ void main() { ), ); - await screenMatchesGolden(tester, 'text-scaling-blockquote'); + await expectLater( + find.byType(MaterialApp).first, + matchesGoldenFileWithPixelAllowance("goldens/text-scaling-blockquote.png", 0), + ); }); }); }); diff --git a/super_editor/test_goldens/super_textfield/super_textfield_scroll_test.dart b/super_editor/test_goldens/super_textfield/super_textfield_scroll_test.dart index abbf2588d5..b4785192dc 100644 --- a/super_editor/test_goldens/super_textfield/super_textfield_scroll_test.dart +++ b/super_editor/test_goldens/super_textfield/super_textfield_scroll_test.dart @@ -3,9 +3,11 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_editor/super_editor.dart'; +import '../test_tools_goldens.dart'; + void main() { group('SuperTextField', () { - testGoldens("multi-line accounts for padding when jumping scroll position down", (tester) async { + testGoldensOnAndroid("multi-line accounts for padding when jumping scroll position down", (tester) async { final controller = AttributedTextEditingController( text: AttributedText(text: "First line\nSecond Line\nThird Line\nFourth Line"), ); @@ -65,7 +67,7 @@ void main() { await screenMatchesGolden(tester, 'super_textfield_scrolled_down'); }); - testGoldens("multi-line accounts for padding when jumping scroll position up", (tester) async { + testGoldensOnAndroid("multi-line accounts for padding when jumping scroll position up", (tester) async { final controller = AttributedTextEditingController( text: AttributedText(text: "First line\nSecond Line\nThird Line\nFourth Line"), ); diff --git a/super_editor/test_goldens/super_textfield/super_textfield_text_alignment_test.dart b/super_editor/test_goldens/super_textfield/super_textfield_text_alignment_test.dart index a4afea20a8..3af42bb752 100644 --- a/super_editor/test_goldens/super_textfield/super_textfield_text_alignment_test.dart +++ b/super_editor/test_goldens/super_textfield/super_textfield_text_alignment_test.dart @@ -5,6 +5,7 @@ import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_editor/super_editor.dart'; import '../../test/test_tools.dart'; +import '../test_tools_goldens.dart'; void main() { // These golden tests are being skipped on macOS because the text seems to be @@ -12,7 +13,7 @@ void main() { group('SuperTextField', () { group('single line', () { group('displays different alignments', () { - testGoldens('(on Android)', (tester) async { + testGoldensOnAndroid('(on Android)', (tester) async { await _pumpScaffold( tester, children: [ @@ -40,7 +41,7 @@ void main() { await screenMatchesGolden(tester, 'super_textfield_alignments_singleline_android'); }, skip: Platform.isMacOS); - testGoldens('(on iOS)', (tester) async { + testGoldensOnAndroid('(on iOS)', (tester) async { await _pumpScaffold( tester, children: [ @@ -68,7 +69,7 @@ void main() { await screenMatchesGolden(tester, 'super_textfield_alignments_singleline_ios'); }, skip: Platform.isMacOS); - testGoldens('(on Desktop)', (tester) async { + testGoldensOnAndroid('(on Desktop)', (tester) async { await _pumpScaffold( tester, children: [ @@ -101,7 +102,7 @@ void main() { group('multi line', () { const multilineText = 'First Line\nSecond Line\nThird Line\nFourth Line'; group('displays different alignments', () { - testGoldens('(on Android)', (tester) async { + testGoldensOnAndroid('(on Android)', (tester) async { await _pumpScaffold( tester, children: [ @@ -129,7 +130,7 @@ void main() { await screenMatchesGolden(tester, 'super_textfield_alignments_multiline_android'); }, skip: Platform.isMacOS); - testGoldens('(on iOS)', (tester) async { + testGoldensOnAndroid('(on iOS)', (tester) async { await _pumpScaffold( tester, children: [ @@ -157,7 +158,7 @@ void main() { await screenMatchesGolden(tester, 'super_textfield_alignments_multiline_ios'); }, skip: Platform.isMacOS); - testGoldens('(on Desktop)', (tester) async { + testGoldensOnAndroid('(on Desktop)', (tester) async { await _pumpScaffold( tester, children: [ diff --git a/super_editor/test_goldens/super_textfield/super_textfield_toolbar_test.dart b/super_editor/test_goldens/super_textfield/super_textfield_toolbar_test.dart index 2e5c0f8a81..8e1bed5161 100644 --- a/super_editor/test_goldens/super_textfield/super_textfield_toolbar_test.dart +++ b/super_editor/test_goldens/super_textfield/super_textfield_toolbar_test.dart @@ -1,13 +1,13 @@ import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_editor/super_editor.dart'; import '../../test/super_textfield/super_textfield_robot.dart'; +import '../test_tools_goldens.dart'; void main() { group('SuperTextField', () { - testGoldens('displays toolbar pointing down', (tester) async { + testGoldensOnAndroid('displays toolbar pointing down', (tester) async { // Pumps a widget tree with a SuperTextField at the bottom of the screen. await _pumpSuperTextfieldToolbarTestApp( tester, @@ -23,10 +23,13 @@ void main() { // Select a word so that the popover toolbar appears. await tester.doubleTapAtSuperTextField(6); - await screenMatchesGolden(tester, 'super_textfield_ios_toolbar_pointing_down'); + await expectLater( + find.byType(MaterialApp), + matchesGoldenFileWithPixelAllowance("goldens/super_textfield_ios_toolbar_pointing_down.png", 1), + ); }); - testGoldens('displays toolbar pointing up', (tester) async { + testGoldensOnAndroid('displays toolbar pointing up', (tester) async { // Pumps a widget tree with a SuperTextField at the top of the screen. await _pumpSuperTextfieldToolbarTestApp( tester, @@ -39,7 +42,10 @@ void main() { // Select a word so that the popover toolbar appears. await tester.doubleTapAtSuperTextField(6); - await screenMatchesGolden(tester, 'super_textfield_ios_toolbar_pointing_up'); + await expectLater( + find.byType(MaterialApp), + matchesGoldenFileWithPixelAllowance("goldens/super_textfield_ios_toolbar_pointing_up.png", 3), + ); }); }); } diff --git a/super_editor/test_goldens/test_tools_goldens.dart b/super_editor/test_goldens/test_tools_goldens.dart new file mode 100644 index 0000000000..252e9c3f57 --- /dev/null +++ b/super_editor/test_goldens/test_tools_goldens.dart @@ -0,0 +1,232 @@ +import 'dart:io'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:golden_toolkit/golden_toolkit.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; + +/// A golden test that configures itself as a Android platform before executing the +/// given [test], and nullifies the Android configuration when the test is done. +@isTest +void testGoldensOnAndroid( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.android; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a iOS platform before executing the +/// given [test], and nullifies the iOS configuration when the test is done. +@isTest +void testGoldensOniOS( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.iOS; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a Mac platform before executing the +/// given [test], and nullifies the Mac configuration when the test is done. +@isTest +void testGoldensOnMac( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.macOS; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a Windows platform before executing the +/// given [test], and nullifies the Windows configuration when the test is done. +@isTest +void testGoldensOnWindows( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.windows; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a Linux platform before executing the +/// given [test], and nullifies the Linux configuration when the test is done. +@isTest +void testGoldensOnLinux( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.linux; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A matcher that expects given content to match the golden file referenced +/// by [key], allowing up to [maxPixelMismatchCount] different pixels before +/// considering the test to be a failure. +/// +/// Typically, the [key] is expected to be a relative file path from the given +/// test file, to the golden file, e.g., "goldens/my-golden-name.png". +/// +/// This matcher can be used by calling it in `expectLater()`, e.g., +/// +/// await expectLater( +/// find.byType(MaterialApp), +/// matchesGoldenFileWithPixelAllowance("goldens/my-golden-name.png", 20), +/// ); +/// +/// Typically, Flutter's golden system describes mismatches in terms of percentages. +/// But percentages are difficult to depend upon. Sometimes a relatively large percentage +/// doesn't matter, and sometimes a tiny percentage is critical. When it comes to ignoring +/// irrelevant mismatches, it's often more convenient to work in terms of pixels. This +/// matcher lets developers specify a maximum pixel mismatch count, instead of relying on +/// percentage differences across the entire golden image. +MatchesGoldenFile matchesGoldenFileWithPixelAllowance(Object key, int maxPixelMismatchCount, {int? version}) { + if (key is Uri) { + return MatchesGoldenFileWithPixelAllowance(key, maxPixelMismatchCount, version); + } else if (key is String) { + return MatchesGoldenFileWithPixelAllowance.forStringPath(key, maxPixelMismatchCount, version); + } + throw ArgumentError('Unexpected type for golden file: ${key.runtimeType}'); +} + +/// A special version of [MatchesGoldenFile] that allows a specified number of +/// pixels to be different between golden files before considering the test to +/// be a failure. +/// +/// Typically, this matcher is expected to be created by calling +/// [matchesGoldenFileWithPixelAllowance]. +class MatchesGoldenFileWithPixelAllowance extends MatchesGoldenFile { + /// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden + /// file at the relative path within the [key] URI. + /// + /// The [key] URI should be a relative path from the executing test's + /// directory to the golden file, e.g., "goldens/my-golden-name.png". + MatchesGoldenFileWithPixelAllowance(super.key, this._maxPixelMismatchCount, [super.version]); + + /// Creates a [MatchesGoldenFileWithPixelAllowance] that looks for a golden + /// file at the relative [path]. + /// + /// The [path] should be relative to the executing test's directory, e.g., + /// "goldens/my-golden-name.png". + MatchesGoldenFileWithPixelAllowance.forStringPath(String path, this._maxPixelMismatchCount, [int? version]) + : super.forStringPath(path, version); + + final int _maxPixelMismatchCount; + + @override + Future matchAsync(dynamic item) async { + // Cache the current goldenFileComparator so we can restore + // it after the test. + final originalComparator = goldenFileComparator; + + try { + goldenFileComparator = PixelDiffGoldenComparator( + (goldenFileComparator as LocalFileComparator).basedir.path, + pixelCount: _maxPixelMismatchCount, + ); + + return await super.matchAsync(item); + } finally { + goldenFileComparator = originalComparator; + } + } +} + +/// A golden file comparator that allows a specified number of pixels +/// to be different between the golden image file and the test image file, and +/// still pass. +class PixelDiffGoldenComparator extends LocalFileComparator { + PixelDiffGoldenComparator( + String testBaseDirectory, { + required int pixelCount, + }) : _testBaseDirectory = testBaseDirectory, + _maxPixelMismatchCount = pixelCount, + super(Uri.parse(testBaseDirectory)); + + @override + Uri get basedir => Uri.parse(_testBaseDirectory); + + /// The file system path to the directory that holds the currently executing + /// Dart test file. + final String _testBaseDirectory; + + /// The maximum number of mismatched pixels for which this pixel test + /// is considered a success/pass. + final int _maxPixelMismatchCount; + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + // Note: the incoming `golden` Uri is a partial path from the currently + // executing test directory to the golden file, e.g., "goldens/my-test.png". + final result = await GoldenFileComparator.compareLists( + imageBytes, + await getGoldenBytes(golden), + ); + + if (result.passed) { + return true; + } + + final diffImage = result.diffs!.entries.first.value; + final pixelCount = diffImage.width * diffImage.height; + final pixelMismatchCount = pixelCount * result.diffPercent; + + if (pixelMismatchCount <= _maxPixelMismatchCount) { + return true; + } + + // Paint the golden diffs and images to failure files. + await generateFailureOutput(result, golden, basedir); + throw FlutterError( + "Pixel test failed. ${result.diffPercent.toStringAsFixed(2)}% diff, $pixelMismatchCount pixel count diff (max allowed pixel mismatch count is $_maxPixelMismatchCount)"); + } + + @override + @protected + Future> getGoldenBytes(Uri golden) async { + final File goldenFile = _getGoldenFile(golden); + if (!goldenFile.existsSync()) { + fail('Could not be compared against non-existent file: "$golden"'); + } + final List goldenBytes = await goldenFile.readAsBytes(); + return goldenBytes; + } + + File _getGoldenFile(Uri golden) => File(path.join(_testBaseDirectory, path.fromUri(golden.path))); +} diff --git a/super_editor/tool/goldens.dart b/super_editor/tool/goldens.dart deleted file mode 100644 index 51f3f4547f..0000000000 --- a/super_editor/tool/goldens.dart +++ /dev/null @@ -1,229 +0,0 @@ -import 'dart:io'; - -// ignore: depend_on_referenced_packages -import 'package:args/command_runner.dart'; - -Future main(List arguments) async { - final runner = CommandRunner("goldens", "A tool to run and update golden tests using docker") - ..addCommand(GoldenTestCommand()) - ..addCommand(UpdateGoldensCommand()); - - try { - await runner.run(arguments); - } on UsageException catch (e) { - stdout.write(e); - } -} - -/// A [Command] which runs golden tests. -/// -/// Usage: `flutter pub run super_editor:goldens test ` -/// -/// Options: -/// -/// `--plain-name "test-name"`: Runs only the tests containing the given value in the test name. -/// -/// The directory is optional. -class GoldenTestCommand extends Command { - GoldenTestCommand() { - argParser.addOption( - 'plain-name', - help: 'A plain-text substring of the names of tests to run', - ); - } - - @override - String get name => 'test'; - - @override - String get description => 'Runs golden tests'; - - @override - Future run() async { - final args = argResults!; - - // Builds the image used to run the container. - // We can build the image even if it already exists. - // Docker will cache each step used in the Dockerfile, so subsequent builds will be faster. - await _buildDockerImage(); - - // Arguments that are placed after 'flutter test test_goldens'. - final cmdArguments = []; - - final name = args['plain-name']; - if (name is String) { - cmdArguments - ..add('--plain-name') - ..add(name); - } - - stdout.writeln('Running golden tests'); - - // Other arguments passed at the end of the command. - // For example, the test directory. - final rest = args.rest; - - final testDirectory = rest.isEmpty // - ? 'test_goldens' - : ''; - - // Runs the container. - // - // --rm: Removes the container when it exists. - // - // -v: Mounts the repo root dir of the host machine into /build directory on the container. - // We need to mount the root to be able to depend on the other packages using the local path. - // - // --workdir: Sets the working directory to /build/super_editor in the container. - await _runProcess( - exe: 'docker', - arguments: [ - 'run', - '--rm', - '-v', - '${Directory.current.path}/../:/build', - '--workdir', - '/build/super_editor', - 'supereditor_golden_tester', - 'flutter', - 'test', - testDirectory, - ...rest, - ...cmdArguments, - ], - description: 'Golden tests', - ); - } -} - -/// A [Command] which updates golden files. -/// -/// Usage: `flutter pub run super_editor:goldens update ` -/// -/// Options: -/// -/// `--plain-name "test-name"`: Update only the tests containing the given value in the test name. -/// -/// The directory is optional. -class UpdateGoldensCommand extends Command { - UpdateGoldensCommand() { - argParser.addOption( - 'plain-name', - help: 'A plain-text substring of the names of tests to run', - ); - } - - @override - String get description => 'Updates golden files'; - - @override - String get name => 'update'; - - @override - Future run() async { - final args = argResults!; - - // Builds the image used to run the container. - // We can build the image even if it already exists. - // Docker will cache each step used in the Dockerfile, so subsequent builds will be faster. - await _buildDockerImage(); - - // Arguments that are placed after 'flutter test test_goldens'. - final cmdArguments = []; - - final name = args['plain-name']; - if (name is String) { - cmdArguments - ..add('--plain-name') - ..add(name); - } - - stdout.writeln('Updating golden files'); - - // Other arguments passed at the end of the command. - // For example, the test directory. - final rest = args.rest; - - final testDirectory = rest.isEmpty // - ? 'test_goldens' - : ''; - - // Runs the container. - // - // --rm: Removes the container when it exists. - // - // -v: Mounts the repo root dir of the host machine into /build directory on the container. - // We need to mount the root to be able to depend on the other packages using the local path. - // - // --workdir: Sets the working directory to /build/super_editor in the container. - await _runProcess( - exe: 'docker', - arguments: [ - 'run', - '--rm', - '-v', - '${Directory.current.path}/../:/build', - '--workdir', - '/build/super_editor', - 'supereditor_golden_tester', - 'flutter', - 'test', - '--update-goldens', - testDirectory, - ...rest, - ...cmdArguments, - ], - description: 'Update goldens', - ); - } -} - -/// Builds a linux docker image to run the tests. -/// -/// The golden_tester.Dockerfile is used to build this image. -Future _buildDockerImage() async { - stdout.write('building image'); - - await _runProcess( - exe: 'docker', - arguments: [ - 'build', - '-f', - './golden_tester.Dockerfile', - '-t', - 'supereditor_golden_tester', - '.', - ], - description: 'Image build', - ); -} - -/// Runs [exe] with the given [arguments]. -/// -/// The child process stdout and stderr are written to the current process stdout. -/// -/// Throws and exception if the process exists with a non-zero exit code. -Future _runProcess({ - required String exe, - required List arguments, - required String description, - String? workingDirectory, -}) async { - final result = await Process.run( - exe, - arguments, - workingDirectory: workingDirectory, - ); - - if (result.stdout != null) { - stdout.write(result.stdout); - } - - if (result.stderr != null) { - stdout.write(result.stderr); - } - - if (result.exitCode != 0) { - throw Exception('$description failed'); - } -} diff --git a/super_text_layout/README_TESTS.md b/super_text_layout/README_TESTS.md index c28f03f13e..50de64f133 100644 --- a/super_text_layout/README_TESTS.md +++ b/super_text_layout/README_TESTS.md @@ -1,35 +1,39 @@ # Running tests -In order to run the golden tests, docker must be installed. +In order to run the golden tests, docker must be installed. Activate the golden_runner with: + +```console +dart pub global activate --source path ../golden_runner +``` ## Run golden tests: ``` # run all tests -flutter pub run tool/goldens test +goldens test # run a single test -flutter pub run tool/goldens test --plain-name "something" +goldens test --plain-name "something" # run all tests in a directory -flutter pub run tool/goldens test test my_dir +goldens test test my_dir # run a single test in a directory -flutter pub run tool/goldens test --plain-name "something" my_dir +goldens test --plain-name "something" my_dir ``` ## Update golden files: ``` # update all goldens -flutter pub run tool/goldens update +goldens update # update all goldens in a directory -flutter pub run tool/goldens update my_dir +goldens update my_dir # update a single golden -flutter pub run tool/goldens update --plain-name "something" +goldens update --plain-name "something" # update a single golden in a directory -flutter pub run tool/goldens update --plain-name "something" my_dir +goldens update --plain-name "something" my_dir ``` diff --git a/super_text_layout/golden_tester.Dockerfile b/super_text_layout/golden_tester.Dockerfile deleted file mode 100644 index 21dc39ab4f..0000000000 --- a/super_text_layout/golden_tester.Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:latest - -ENV FLUTTER_HOME=${HOME}/sdks/flutter -ENV PATH ${PATH}:${FLUTTER_HOME}/bin:${FLUTTER_HOME}/bin/cache/dart-sdk/bin - -USER root - -RUN apt update - -RUN apt install -y git curl unzip -RUN git clone https://github.com/flutter/flutter.git ${FLUTTER_HOME} - -RUN flutter doctor \ No newline at end of file diff --git a/super_text_layout/pubspec.yaml b/super_text_layout/pubspec.yaml index a4e8aef870..931aa22576 100644 --- a/super_text_layout/pubspec.yaml +++ b/super_text_layout/pubspec.yaml @@ -28,6 +28,9 @@ dev_dependencies: flutter_lints: ^2.0.1 golden_toolkit: ^0.13.0 args: ^2.3.1 + meta: ^1.8.0 + golden_runner: + path: ../golden_runner flutter: # no Flutter configuration diff --git a/super_text_layout/test_goldens/caret_layer_test.dart b/super_text_layout/test_goldens/caret_layer_test.dart index db1b5cd269..99b154b9b6 100644 --- a/super_text_layout/test_goldens/caret_layer_test.dart +++ b/super_text_layout/test_goldens/caret_layer_test.dart @@ -4,13 +4,14 @@ import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_text_layout/super_text_layout.dart'; import 'test_tools.dart'; +import 'test_tools_goldens.dart'; const primaryCaretStyle = CaretStyle(color: Colors.black); void main() { group("Caret layer", () { group("with a single caret", () { - testGoldens("paints a normal caret", (tester) async { + testGoldensOnAndroid("paints a normal caret", (tester) async { await pumpThreeLinePlainSuperText( tester, aboveBuilder: (context, TextLayout textLayout) { @@ -30,7 +31,7 @@ void main() { await screenMatchesGolden(tester, "CaretLayer_single-caret_normal"); }); - testGoldens("paints caret styles", (tester) async { + testGoldensOnAndroid("paints caret styles", (tester) async { await pumpThreeLinePlainSuperText( tester, aboveBuilder: (context, TextLayout textLayout) { @@ -56,7 +57,7 @@ void main() { }); group("with multiple carets", () { - testGoldens("paints multiple carets", (tester) async { + testGoldensOnAndroid("paints multiple carets", (tester) async { await pumpThreeLinePlainSuperText( tester, aboveBuilder: (context, TextLayout textLayout) { @@ -88,7 +89,7 @@ void main() { await screenMatchesGolden(tester, "CaretLayer_multi-caret"); }); - testGoldens("paints two carets at the same position", (tester) async { + testGoldensOnAndroid("paints two carets at the same position", (tester) async { await pumpThreeLinePlainSuperText( tester, aboveBuilder: (context, TextLayout textLayout) { diff --git a/super_text_layout/test_goldens/super_text_layers_test.dart b/super_text_layout/test_goldens/super_text_layers_test.dart index 3266c0a422..004d8d43b6 100644 --- a/super_text_layout/test_goldens/super_text_layers_test.dart +++ b/super_text_layout/test_goldens/super_text_layers_test.dart @@ -3,11 +3,12 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:golden_toolkit/golden_toolkit.dart'; import 'test_tools.dart'; +import 'test_tools_goldens.dart'; void main() { group("SuperText", () { group("builds layers", () { - testGoldens("that can paint line boxes", (tester) async { + testGoldensOnAndroid("that can paint line boxes", (tester) async { await pumpThreeLinePlainSuperText(tester, beneathBuilder: (context, textLayout) { final lineCount = textLayout.getLineCount(); final lineRects = []; @@ -42,7 +43,7 @@ void main() { await screenMatchesGolden(tester, "SuperText_layers_line-boxes"); }); - testGoldens("that can paint character boxes", (tester) async { + testGoldensOnAndroid("that can paint character boxes", (tester) async { await pumpThreeLinePlainSuperText(tester, beneathBuilder: (context, textLayout) { final characterRects = []; final characterColors = []; @@ -70,7 +71,7 @@ void main() { await screenMatchesGolden(tester, "SuperText_layers_character-boxes"); }); - testGoldens("that can paint character box outlines", (tester) async { + testGoldensOnAndroid("that can paint character box outlines", (tester) async { await pumpThreeLinePlainSuperText(tester, beneathBuilder: (context, textLayout) { final characterRects = []; @@ -98,7 +99,7 @@ void main() { await screenMatchesGolden(tester, "SuperText_layers_character-box-outlines"); }); - testGoldens("that can paint carets", (tester) async { + testGoldensOnAndroid("that can paint carets", (tester) async { await pumpThreeLinePlainSuperText(tester, beneathBuilder: (context, textLayout) { const textPosition = TextPosition(offset: 115); final caretOffset = textLayout.getOffsetForCaret(textPosition); diff --git a/super_text_layout/test_goldens/super_text_test.dart b/super_text_layout/test_goldens/super_text_test.dart index 851b9a9125..c7fca23717 100644 --- a/super_text_layout/test_goldens/super_text_test.dart +++ b/super_text_layout/test_goldens/super_text_test.dart @@ -3,15 +3,17 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_text_layout/super_text_layout.dart'; +import 'test_tools_goldens.dart'; + void main() { group("SuperText", () { group("text layout", () { - testGoldens("renders a visual reference for non-visual tests", (tester) async { + testGoldensOnAndroid("renders a visual reference for non-visual tests", (tester) async { await _pumpThreeLinePlainText(tester); await screenMatchesGolden(tester, "SuperText-reference-render"); }); - testGoldens("applies textScaleFactor", (tester) async { + testGoldensOnAndroid("applies textScaleFactor", (tester) async { await tester.pumpWidget( _buildScaffold( // ignore: prefer_const_constructors diff --git a/super_text_layout/test_goldens/test_tools_goldens.dart b/super_text_layout/test_goldens/test_tools_goldens.dart new file mode 100644 index 0000000000..b3015100ac --- /dev/null +++ b/super_text_layout/test_goldens/test_tools_goldens.dart @@ -0,0 +1,94 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:golden_toolkit/golden_toolkit.dart'; +import 'package:meta/meta.dart'; + +/// A golden test that configures itself as a Android platform before executing the +/// given [test], and nullifies the Android configuration when the test is done. +@isTest +void testGoldensOnAndroid( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.android; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a iOS platform before executing the +/// given [test], and nullifies the iOS configuration when the test is done. +@isTest +void testGoldensOniOS( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.iOS; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a Mac platform before executing the +/// given [test], and nullifies the Mac configuration when the test is done. +@isTest +void testGoldensOnMac( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.macOS; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a Windows platform before executing the +/// given [test], and nullifies the Windows configuration when the test is done. +@isTest +void testGoldensOnWindows( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.windows; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} + +/// A golden test that configures itself as a Linux platform before executing the +/// given [test], and nullifies the Linux configuration when the test is done. +@isTest +void testGoldensOnLinux( + String description, + WidgetTesterCallback test, { + bool skip = false, +}) { + testGoldens(description, (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.linux; + try { + await test(tester); + } finally { + debugDefaultTargetPlatformOverride = null; + } + }, skip: skip); +} diff --git a/super_text_layout/test_goldens/text_selection_layer_test.dart b/super_text_layout/test_goldens/text_selection_layer_test.dart index b9bca61032..08d16e586a 100644 --- a/super_text_layout/test_goldens/text_selection_layer_test.dart +++ b/super_text_layout/test_goldens/text_selection_layer_test.dart @@ -4,6 +4,7 @@ import 'package:golden_toolkit/golden_toolkit.dart'; import 'package:super_text_layout/super_text_layout.dart'; import 'test_tools.dart'; +import 'test_tools_goldens.dart'; void main() { group("Text selection layer", () { @@ -11,7 +12,7 @@ void main() { color: defaultSelectionColor, ); - testGoldens("paints a full text selection", (tester) async { + testGoldensOnAndroid("paints a full text selection", (tester) async { await pumpThreeLinePlainSuperText( tester, beneathBuilder: (context, textLayout) { @@ -29,7 +30,7 @@ void main() { await screenMatchesGolden(tester, "TextSelectionLayer_full-selection"); }); - testGoldens("paints a partial text selection", (tester) async { + testGoldensOnAndroid("paints a partial text selection", (tester) async { await pumpThreeLinePlainSuperText( tester, beneathBuilder: (context, textLayout) { @@ -47,7 +48,7 @@ void main() { await screenMatchesGolden(tester, "TextSelectionLayer_partial-selection"); }); - testGoldens("paints an empty highlight when text is empty", (tester) async { + testGoldensOnAndroid("paints an empty highlight when text is empty", (tester) async { await pumpEmptySuperText( tester, beneathBuilder: (context, textLayout) { @@ -61,7 +62,7 @@ void main() { await screenMatchesGolden(tester, "TextSelectionLayer_small-highlight-when-empty"); }); - testGoldens("paints no selection when text is empty", (tester) async { + testGoldensOnAndroid("paints no selection when text is empty", (tester) async { await pumpEmptySuperText( tester, beneathBuilder: (context, textLayout) { diff --git a/super_text_layout/tool/goldens.dart b/super_text_layout/tool/goldens.dart deleted file mode 100644 index 395d4ba874..0000000000 --- a/super_text_layout/tool/goldens.dart +++ /dev/null @@ -1,229 +0,0 @@ -import 'dart:io'; - -// ignore: depend_on_referenced_packages -import 'package:args/command_runner.dart'; - -Future main(List arguments) async { - final runner = CommandRunner("goldens", "A tool to run and update golden tests using docker") - ..addCommand(GoldenTestCommand()) - ..addCommand(UpdateGoldensCommand()); - - try { - await runner.run(arguments); - } on UsageException catch (e) { - stdout.write(e); - } -} - -/// A [Command] which runs golden tests. -/// -/// Usage: `flutter pub run super_text_layout:goldens test ` -/// -/// Options: -/// -/// `--plain-name "test-name"`: Runs only the tests containing the given value in the test name. -/// -/// The directory is optional. -class GoldenTestCommand extends Command { - GoldenTestCommand() { - argParser.addOption( - 'plain-name', - help: 'A plain-text substring of the names of tests to run', - ); - } - - @override - String get name => 'test'; - - @override - String get description => 'Runs golden tests'; - - @override - Future run() async { - final args = argResults!; - - // Builds the image used to run the container. - // We can build the image even if it already exists. - // Docker will cache each step used in the Dockerfile, so subsequent builds will be faster. - await _buildDockerImage(); - - // Arguments that are placed after 'flutter test test_goldens'. - final cmdArguments = []; - - final name = args['plain-name']; - if (name is String) { - cmdArguments - ..add('--plain-name') - ..add(name); - } - - stdout.writeln('Running golden tests'); - - // Other arguments passed at the end of the command. - // For example, the test directory. - final rest = args.rest; - - final testDirectory = rest.isEmpty // - ? 'test_goldens' - : ''; - - // Runs the container. - // - // --rm: Removes the container when it exists. - // - // -v: Mounts the repo root dir of the host machine into /build directory on the container. - // We need to mount the root to be able to depend on the other packages using the local path. - // - // --workdir: Sets the working directory to /build/super_text_layout in the container. - await _runProcess( - exe: 'docker', - arguments: [ - 'run', - '--rm', - '-v', - '${Directory.current.path}/../:/build', - '--workdir', - '/build/super_text_layout', - 'supereditor_golden_tester', - 'flutter', - 'test', - testDirectory, - ...rest, - ...cmdArguments, - ], - description: 'Golden tests', - ); - } -} - -/// A [Command] which updates golden files. -/// -/// Usage: `flutter pub run super_text_layout:goldens update ` -/// -/// Options: -/// -/// `--plain-name "test-name"`: Update only the tests containing the given value in the test name. -/// -/// The directory is optional. -class UpdateGoldensCommand extends Command { - UpdateGoldensCommand() { - argParser.addOption( - 'plain-name', - help: 'A plain-text substring of the names of tests to run', - ); - } - - @override - String get description => 'Updates golden files'; - - @override - String get name => 'update'; - - @override - Future run() async { - final args = argResults!; - - // Builds the image used to run the container. - // We can build the image even if it already exists. - // Docker will cache each step used in the Dockerfile, so subsequent builds will be faster. - await _buildDockerImage(); - - // Arguments that are placed after 'flutter test test_goldens'. - final cmdArguments = []; - - final name = args['plain-name']; - if (name is String) { - cmdArguments - ..add('--plain-name') - ..add(name); - } - - stdout.writeln('Updating golden files'); - - // Other arguments passed at the end of the command. - // For example, the test directory. - final rest = args.rest; - - final testDirectory = rest.isEmpty // - ? 'test_goldens' - : ''; - - // Runs the container. - // - // --rm: Removes the container when it exists. - // - // -v: Mounts the repo root dir of the host machine into /build directory on the container. - // We need to mount the root to be able to depend on the other packages using the local path. - // - // --workdir: Sets the working directory to /build/super_text_layout in the container. - await _runProcess( - exe: 'docker', - arguments: [ - 'run', - '--rm', - '-v', - '${Directory.current.path}/../:/build', - '--workdir', - '/build/super_text_layout', - 'supereditor_golden_tester', - 'flutter', - 'test', - '--update-goldens', - testDirectory, - ...rest, - ...cmdArguments, - ], - description: 'Update goldens', - ); - } -} - -/// Builds a linux docker image to run the tests. -/// -/// The golden_tester.Dockerfile is used to build this image. -Future _buildDockerImage() async { - stdout.write('building image'); - - await _runProcess( - exe: 'docker', - arguments: [ - 'build', - '-f', - './golden_tester.Dockerfile', - '-t', - 'supereditor_golden_tester', - '.', - ], - description: 'Image build', - ); -} - -/// Runs [exe] with the given [arguments]. -/// -/// The child process stdout and stderr are written to the current process stdout. -/// -/// Throws and exception if the process exists with a non-zero exit code. -Future _runProcess({ - required String exe, - required List arguments, - required String description, - String? workingDirectory, -}) async { - final result = await Process.run( - exe, - arguments, - workingDirectory: workingDirectory, - ); - - if (result.stdout != null) { - stdout.write(result.stdout); - } - - if (result.stderr != null) { - stdout.write(result.stderr); - } - - if (result.exitCode != 0) { - throw Exception('$description failed'); - } -}