Adapt golden testing framework from pub client

pubspec.yaml

@@ -44,6 +44,7 @@ dev_dependencies:
   json_serializable: ^6.9.5
   shelf: ^1.4.2
   source_gen: ^4.0.0
+  stack_trace: ^1.12.1
   tar: ^2.0.0
   test_descriptor: ^2.0.2
   test_process: ^2.1.1

test/bin_pana_test.dart

@@ -5,40 +5,29 @@
 import 'dart:io';
 
 import 'package:io/io.dart';
-import 'package:path/path.dart' as p;
 import 'package:test/test.dart';
-import 'package:test_process/test_process.dart';
 
 import 'golden_file.dart';
 
-final helpGoldenPath = p.join('test', 'goldens', 'help.txt');
-
 void main() {
   // This is really two tests in one, because the second one depends on the
   // golden file from the first.
-  test(
+  testWithGolden(
     'run with bad option shows help text. Help text is included in readme ',
-    () async {
-      var process = await TestProcess.start(
-        p.join(p.dirname(Platform.resolvedExecutable), 'dart'),
-        ['pub', 'run', 'pana', '--monkey'],
-      );
-
-      var output = await process.stdoutStream().join('\n');
-
-      const prefix = 'Could not find an option named "--monkey".\n\n';
-
-      expect(output, startsWith(prefix));
-      expectMatchesGoldenFile(output.substring(prefix.length), helpGoldenPath);
-
-      await process.shouldExit(ExitCode.usage.code);
-
-      var readme = File('README.md');
-      expect(
-        readme.readAsStringSync().replaceAll('\r\n', '\n'),
-        contains(
-          '```\n${File(helpGoldenPath).readAsStringSync().replaceAll('\r\n', '\n')}\n```',
+    (goldenContext) async {
+      final readme = File('README.md').readAsStringSync();
+      final helpText = RegExp(
+        r'```\n(Usage:.*)\n```',
+        multiLine: true,
+        dotAll: true,
+      ).firstMatch(readme)![1]!;
+      await goldenContext.run(
+        [Platform.resolvedExecutable, 'run', 'pana', '--monkey'],
+        stdoutExpectation: allOf(
+          contains('Could not find an option named "--monkey".\n\n'),
+          contains(helpText),
         ),
+        exitCodeExpectation: ExitCode.usage.code,
       );
     },
   );

test/end2end_test.dart

@@ -12,15 +12,14 @@ import 'package:test/test.dart';
 import 'env_utils.dart';
 import 'golden_file.dart';
 
-final _goldenDir = p.join('test', 'goldens', 'end2end');
+final _testDataDir = p.join('test', 'testData');
 
 void main() {
   void verifyPackage(
     String package,
     String version, {
     bool skipDartdoc = false,
   }) {
-    final filename = '$package-$version.json';
     group('end2end: $package $version', () {
       late TestEnv testEnv;
       late final Map<String, Object?> actualMap;
@@ -102,7 +101,7 @@ void main() {
         await testEnv.close();
       });
 
-      test('matches known good', () {
+      testWithGolden('$package $version report', (context) {
         void removeDependencyDetails(Map<String, dynamic> map) {
           if (map.containsKey('pkgResolution') &&
               (map['pkgResolution'] as Map).containsKey('dependencies')) {
@@ -128,9 +127,6 @@ void main() {
           r'Error on line 5, column 1 of $TEMPDIR/pubspec.yaml',
         );
 
-        final jsonGoldenFile = GoldenFile(p.join(_goldenDir, filename));
-        jsonGoldenFile.writeContentIfNotExists(jsonNoTempDir);
-
         final jsonReport = actualMap['report'] as Map<String, Object?>?;
         if (jsonReport != null) {
           final report = Report.fromJson(jsonReport);
@@ -140,16 +136,12 @@ void main() {
                     '## ${s.grantedPoints}/${s.maxPoints} ${s.title}\n\n${s.summary}',
               )
               .join('\n\n');
-          // For readability we output the report in its own file.
-          final reportGoldenFile = GoldenFile(
-            p.join(_goldenDir, '${filename}_report.md'),
+          context.expectSection(
+            renderedSections.replaceAll('\r\n', '\n'),
+            sectionTitle: 'rendered report',
           );
-          reportGoldenFile.writeContentIfNotExists(renderedSections);
-          reportGoldenFile.expectContent(renderedSections);
         }
-
-        // note: golden file expectations happen after content is already written
-        jsonGoldenFile.expectContent(jsonNoTempDir);
+        context.expectSection(jsonNoTempDir, sectionTitle: 'json report');
 
         var summary = Summary.fromJson(actualMap);
 
@@ -233,7 +225,7 @@ void main() {
 }
 
 Future<DateTime> _detectGoldenLastModified() async {
-  final timestampFile = File(p.join(_goldenDir, '__timestamp.txt'));
+  final timestampFile = File(p.join(_testDataDir, '__timestamp.txt'));
   await timestampFile.parent.create(recursive: true);
   if (timestampFile.existsSync()) {
     final content = await timestampFile.readAsString();

test/golden_file.dart

@@ -1,50 +1,220 @@
-// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:io';
 
+import 'package:path/path.dart' as p;
+import 'package:stack_trace/stack_trace.dart' show Trace;
 import 'package:test/test.dart';
 
-/// Will test [actual] against the contests of the file at [goldenFilePath].
-///
-/// If the file doesn't exist, the file is instead created containing [actual].
-void expectMatchesGoldenFile(String actual, String goldenFilePath) {
-  final goldenFile = GoldenFile(goldenFilePath);
-  goldenFile.writeContentIfNotExists(actual);
-  goldenFile.expectContent(actual);
-}
+final _isCI = () {
+  final p = RegExp(r'^1|(?:true)$', caseSensitive: false);
+  final ci = Platform.environment['CI'];
+  return ci != null && ci.isNotEmpty && p.hasMatch(ci);
+}();
 
-/// Access to a file that contains the expected output of a process.
-class GoldenFile {
-  final String path;
-  late final File _file;
-  late final bool _didExists;
-  late final String? _oldContent;
-
-  GoldenFile(this.path) {
-    _file = File(path);
-    _didExists = _file.existsSync();
-    _oldContent = _didExists ? _file.readAsStringSync() : null;
-  }
+/// Find the current `_test.dart` filename invoked from stack-trace.
+String _findCurrentTestFilename() => Trace.current().frames
+    .lastWhere(
+      (frame) =>
+          frame.uri.isScheme('file') &&
+          p.basename(frame.uri.toFilePath()).endsWith('_test.dart'),
+    )
+    .uri
+    .toFilePath();
+
+class GoldenTestContext {
+  static const _endOfSection =
+      ''
+      '--------------------------------'
+      ' END OF OUTPUT '
+      '---------------------------------\n\n';
+
+  final File _goldenFile;
+  final String _header;
+  final _results = <String>[];
+  late bool _shouldRegenerateGolden;
+  final bool colors;
+  bool _generatedNewData = false; // track if new data is generated
+  int _nextSectionIndex = 0;
+
+  GoldenTestContext._(
+    String currentTestFile,
+    String testName, {
+    required this.colors,
+  }) : _goldenFile = File(
+         p.join(
+           'test',
+           'testdata',
+           'goldens',
+           p.relative(
+             currentTestFile.replaceAll(RegExp(r'\.dart$'), ''),
+             from: p.join(p.current, 'test'),
+           ),
+           // Sanitize the name, and add .ans or .txt.
+           '${testName.replaceAll(RegExp(r'[<>:"/\|?*%#]'), '~')}.'
+               '${colors ? 'ans' : 'txt'}',
+         ),
+       ),
+       _header = '# GENERATED BY: ${p.relative(currentTestFile)}\n\n';
+
+  String get _goldenFilePath => _goldenFile.path;
 
-  void writeContentIfNotExists(String content) {
-    if (_didExists) return;
-    _file.createSync(recursive: true);
-    _file.writeAsStringSync(content);
+  void _readGoldenFile() {
+    if (RegExp(
+          r'^1|(?:true)$',
+          caseSensitive: false,
+        ).hasMatch(Platform.environment['_WRITE_GOLDEN'] ?? '') ||
+        !_goldenFile.existsSync()) {
+      _shouldRegenerateGolden = true;
+    } else {
+      _shouldRegenerateGolden = false;
+      // Read the golden file for this test
+      var text = _goldenFile.readAsStringSync().replaceAll('\r\n', '\n');
+      // Strip header line
+      if (text.startsWith('#') && text.contains('\n\n')) {
+        text = text.substring(text.indexOf('\n\n') + 2);
+      }
+      _results.addAll(text.split(_endOfSection));
+    }
   }
 
-  void expectContent(String actual) {
-    if (_didExists) {
+  /// Expect section [sectionIndex] to match [actual].
+  void _expectSection(int sectionIndex, String actual) {
+    if (!_shouldRegenerateGolden &&
+        _results.length > sectionIndex &&
+        _results[sectionIndex].isNotEmpty) {
       expect(
-        actual.replaceAll('\r\n', '\n'),
-        equals(_oldContent!.replaceAll('\r\n', '\n')),
-        reason:
-            'goldenFilePath: "$path". '
-            'To update the expectation delete this file and rerun the test.',
+        actual,
+        equals(_results[sectionIndex]),
+        reason: 'Expect matching section $sectionIndex from "$_goldenFilePath"',
       );
     } else {
-      fail('Golden file $path was recreated!');
+      while (_results.length <= sectionIndex) {
+        _results.add('');
+      }
+      _results[sectionIndex] = actual;
+      _generatedNewData = true;
+    }
+  }
+
+  void _writeGoldenFile() {
+    // If we generated new data, then we need to write a new file, and fail the
+    // test case, or mark it as skipped.
+    if (_generatedNewData) {
+      // This enables writing the updated file when run in otherwise hermetic
+      // settings.
+      //
+      // This is to make updating the golden files easier in a bazel environment
+      // See https://docs.bazel.build/versions/2.0.0/user-manual.html#run .
+      var goldenFile = _goldenFile;
+      final workspaceDirectory =
+          Platform.environment['BUILD_WORKSPACE_DIRECTORY'];
+      if (workspaceDirectory != null) {
+        goldenFile = File(p.join(workspaceDirectory, _goldenFilePath));
+      }
+      goldenFile
+        ..createSync(recursive: true)
+        ..writeAsStringSync(_header + _results.join(_endOfSection));
+
+      // If running in CI we should fail if the golden file doesn't already
+      // exist, or is missing entries.
+      // This typically happens if we forgot to commit a file to git.
+      if (_isCI) {
+        fail(
+          'Missing golden file: "$_goldenFilePath", '
+          'try running tests again and commit the file',
+        );
+      } else {
+        // If not running in CI, then we consider the test as skipped, we've
+        // generated the file, but the user should run the tests again.
+        // Or push to CI in which case we'll run the tests again anyways.
+        markTestSkipped(
+          'Generated golden file: "$_goldenFilePath" instead of running test',
+        );
+      }
     }
   }
+
+  /// Expect the next section in the golden file to match [actual].
+  ///
+  /// This will create the section if it is missing.
+  ///
+  /// **Warning**: Take care when using this in an async context, sections are
+  /// numbered based on the other in which calls are made. Hence, ensure
+  /// consistent ordering of calls.
+  void expectSection(String actual, {String? sectionTitle}) {
+    _expectSection(_nextSectionIndex++, '''
+## Section ${sectionTitle ?? _nextSectionIndex}
+$actual''');
+  }
+
+  /// Run [command] with [environment] in [workingDirectory], and
+  /// log stdout/stderr and exit code to golden file.
+  Future<void> run(
+    List<String> command, {
+    Map<String, String>? environment,
+    String? workingDirectory,
+    String? stdin,
+    Matcher? stdoutExpectation,
+    Matcher? stderrExpectation,
+    Object? exitCodeExpectation = 0,
+  }) async {
+    final sectionIndex = _nextSectionIndex++;
+    final executable = command.first;
+    final args = command.skip(1).toList();
+    final result = await Process.run(
+      executable,
+      args,
+      environment: environment,
+      workingDirectory: workingDirectory,
+    );
+    final s = StringBuffer();
+    s.writeln('## Section $sectionIndex');
+    s.writeln('Run \$ ${p.basename(executable)} ${args.join(' ')}');
+    s.writeln('Exit code: ${result.exitCode}');
+    s.write('stdout: ${result.stdout}');
+    s.write('stderr: ${result.stderr}');
+    if (stdoutExpectation != null) {
+      expect(result.stdout, stdoutExpectation);
+    }
+    if (stderrExpectation != null) {
+      expect(result.stderr, stderrExpectation);
+    }
+    _expectSection(sectionIndex, s.toString());
+  }
+}
+
+/// Create a [test] with [GoldenTestContext] which allows running golden tests.
+///
+/// This will create a golden file containing output of calls to:
+///  * [GoldenTestContext.run]
+///
+/// The golden file with the recorded output will be created at:
+///   `test/testdata/goldens/path/to/myfile_test/<name>.txt`
+/// , when `path/to/myfile_test.dart` is the `_test.dart` file from which this
+/// function is called.
+///
+/// If [colors] is `true` the file will be created with an `.ans` extension that
+/// indicates ANSI colors will be used.
+///
+/// Such a file can eg. be viewed in vscode with this plugin:
+/// https://marketplace.visualstudio.com/items?itemName=iliazeus.vscode-ansi
+void testWithGolden(
+  String name,
+  FutureOr<void> Function(GoldenTestContext ctx) fn, {
+  bool colors = false,
+}) {
+  final ctx = GoldenTestContext._(
+    _findCurrentTestFilename(),
+    name,
+    colors: colors,
+  );
+  test(name, () async {
+    ctx._readGoldenFile();
+    await fn(ctx);
+    ctx._writeGoldenFile();
+  });
 }