Batch writes

build_runner/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.4.14-wip
+
+- Write generated assets at the end of a build to avoid invalidating other
+  tools with a file watcher multiple times.
+
 ## 2.4.13
 
 - Bump the min sdk to 3.5.0.

build_runner/lib/src/watcher/delete_writer.dart

@@ -29,4 +29,9 @@ class OnDeleteWriter implements RunnerAssetWriter {
   Future writeAsString(AssetId id, String contents,
           {Encoding encoding = utf8}) =>
       _writer.writeAsString(id, contents, encoding: encoding);
+
+  @override
+  Future<void> completeBuild() async {
+    await _writer.completeBuild();
+  }
 }

build_runner/pubspec.yaml

@@ -1,8 +1,8 @@
 name: build_runner
-version: 2.4.13
+version: 2.4.14-wip
 description: A build system for Dart code generation and modular compilation.
 repository: https://github.com/dart-lang/build/tree/master/build_runner
-resolution: workspace
+#resolution: workspace
 
 environment:
   sdk: ^3.5.0
@@ -20,7 +20,7 @@ dependencies:
   build_config: ">=1.1.0 <1.2.0"
   build_daemon: ^4.0.0
   build_resolvers: ^2.0.0
-  build_runner_core: ^7.2.0
+  build_runner_core: ^8.0.0-wip
   code_builder: ^4.2.0
   collection: ^1.15.0
   crypto: ^3.0.0
@@ -53,10 +53,15 @@ dev_dependencies:
     path: ../_test_common
   build_test: ^2.0.0
   build_web_compilers: ^4.0.0
+  dart_flutter_team_lints: ^3.1.0
   stream_channel: ^2.0.0
   test: ^1.25.5
   test_descriptor: ^2.0.0
   test_process: ^2.0.0
 
+dependency_overrides:
+  build_runner_core:
+    path: ../build_runner_core
+
 topics:
  - build-runner

build_runner_core/CHANGELOG.md

@@ -1,5 +1,9 @@
-## 7.3.3-wip
+## 8.0.0-wip
 
+- __Breaking__: Add `completeBuild` to `RunnerAssetWriter`, a method expected
+  to be called by the build system at the end of a completed build.
+- Add `wrapInBatch` to obtain a reader/writer pair that will batch writes
+  before flushing them at the end of a build.
 - Bump the min sdk to 3.6.0-dev.228.
 - Require analyzer ^6.9.0.
 - Fix analyzer deprecations.

build_runner_core/lib/build_runner_core.dart

@@ -4,6 +4,7 @@
 
 export 'package:build/build.dart' show PostProcessBuildStep, PostProcessBuilder;
 
+export 'src/asset/batch.dart' show wrapInBatch;
 export 'src/asset/file_based.dart';
 export 'src/asset/finalized_reader.dart';
 export 'src/asset/reader.dart' show RunnerAssetReader;

build_runner_core/lib/src/asset/batch.dart

@@ -0,0 +1,163 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+
+import 'package:build/build.dart';
+import 'package:glob/glob.dart';
+import 'package:meta/meta.dart';
+
+import '../environment/io_environment.dart';
+import 'reader.dart';
+import 'writer.dart';
+
+/// A batch of file system writes that should be committed at once instead of
+/// when [AssetWriter.writeAsBytes] or [AssetWriter.writeAsString] is called.
+///
+/// During a typical build run emitting generated files one-by-one, it's
+/// possible that other running tools such as an analysis server will have to
+/// re-analyze incomplete states multiple times.
+/// By storing pending outputs in memory first and then committing them at the
+/// end of the build, we have a better view over that needs to happen.
+///
+/// The default [IOEnvironment] uses readers and writes that are batch-aware
+/// outside of low-memory mode.
+final class _FileSystemWriteBatch {
+  final Map<AssetId, _PendingFileState> _pendingWrites = {};
+
+  _FileSystemWriteBatch._();
+
+  Future<void> completeWrites(RunnerAssetWriter writer) async {
+    await Future.wait(_pendingWrites.keys.map((id) async {
+      final pending = _pendingWrites[id]!;
+
+      if (pending.content case final content?) {
+        await writer.writeAsBytes(id, content);
+      } else {
+        await writer.delete(id);
+      }
+    }));
+
+    _pendingWrites.clear();
+  }
+}
+
+/// Wraps a pair of a [RunnerAssetReader] with path-prividing capabilities and
+/// a [RunnerAssetWriter] into a pair of readers and writers that will
+/// internally buffer writes and only flush them in
+/// [RunnerAssetWriter.completeBuild].
+///
+/// The returned reader will see pending writes by the returned writer before
+/// they are flushed to the file system.
+(RunnerAssetReader, RunnerAssetWriter) wrapInBatch({
+  required RunnerAssetReader reader,
+  required PathProvidingAssetReader pathProvidingReader,
+  required RunnerAssetWriter writer,
+}) {
+  final batch = _FileSystemWriteBatch._();
+
+  return (
+    BatchReader(reader, pathProvidingReader, batch),
+    BatchWriter(writer, batch),
+  );
+}
+
+final class _PendingFileState {
+  final List<int>? content;
+
+  const _PendingFileState(this.content);
+
+  bool get isDeleted => content == null;
+}
+
+@internal
+final class BatchReader extends AssetReader
+    implements RunnerAssetReader, PathProvidingAssetReader {
+  final RunnerAssetReader _inner;
+  final PathProvidingAssetReader _innerPathProviding;
+  final _FileSystemWriteBatch _batch;
+
+  BatchReader(this._inner, this._innerPathProviding, this._batch);
+
+  _PendingFileState? _stateFor(AssetId id) {
+    return _batch._pendingWrites[id];
+  }
+
+  @override
+  Future<bool> canRead(AssetId id) async {
+    if (_stateFor(id) case final state?) {
+      return !state.isDeleted;
+    } else {
+      return await _inner.canRead(id);
+    }
+  }
+
+  @override
+  Stream<AssetId> findAssets(Glob glob, {String? package}) {
+    return _inner
+        .findAssets(glob, package: package)
+        .where((asset) => _stateFor(asset)?.isDeleted != true);
+  }
+
+  @override
+  String pathTo(AssetId id) {
+    return _innerPathProviding.pathTo(id);
+  }
+
+  @override
+  Future<List<int>> readAsBytes(AssetId id) async {
+    if (_stateFor(id) case final state?) {
+      if (state.isDeleted) {
+        throw AssetNotFoundException(id);
+      } else {
+        return state.content!;
+      }
+    } else {
+      return await _inner.readAsBytes(id);
+    }
+  }
+
+  @override
+  Future<String> readAsString(AssetId id, {Encoding encoding = utf8}) async {
+    if (_stateFor(id) case final state?) {
+      if (state.isDeleted) {
+        throw AssetNotFoundException(id);
+      } else {
+        return encoding.decode(state.content!);
+      }
+    } else {
+      return await _inner.readAsString(id, encoding: encoding);
+    }
+  }
+}
+
+@internal
+final class BatchWriter extends RunnerAssetWriter {
+  final RunnerAssetWriter _inner;
+  final _FileSystemWriteBatch _batch;
+
+  BatchWriter(this._inner, this._batch);
+
+  @override
+  Future delete(AssetId id) async {
+    _batch._pendingWrites[id] = const _PendingFileState(null);
+  }
+
+  @override
+  Future<void> writeAsBytes(AssetId id, List<int> bytes) async {
+    _batch._pendingWrites[id] = _PendingFileState(bytes);
+  }
+
+  @override
+  Future<void> writeAsString(AssetId id, String contents,
+      {Encoding encoding = utf8}) async {
+    _batch._pendingWrites[id] = _PendingFileState(encoding.encode(contents));
+  }
+
+  @override
+  Future<void> completeBuild() async {
+    await _batch.completeWrites(_inner);
+  }
+}

build_runner_core/lib/src/asset/build_cache.dart

@@ -84,6 +84,9 @@ class BuildCacheWriter implements RunnerAssetWriter {
   @override
   Future delete(AssetId id) =>
       _delegate.delete(_cacheLocation(id, _assetGraph, _rootPackage));
+
+  @override
+  Future<void> completeBuild() async {}
 }
 
 AssetId _cacheLocation(AssetId id, AssetGraph assetGraph, String rootPackage) {

build_runner_core/lib/src/asset/file_based.dart

@@ -107,6 +107,9 @@ class FileBasedAssetWriter implements RunnerAssetWriter {
       }
     });
   }
+
+  @override
+  Future<void> completeBuild() async {}
 }
 
 /// Returns the path to [id] for a given [packageGraph].

build_runner_core/lib/src/asset/writer.dart

@@ -11,4 +11,10 @@ typedef OnDelete = void Function(AssetId id);
 
 abstract class RunnerAssetWriter implements AssetWriter {
   Future delete(AssetId id);
+
+  /// Called after each completed build.
+  ///
+  /// Some [RunnerAssetWriter] implementations may buffer completed writes
+  /// internally and flush them in [completeBuild].
+  Future<void> completeBuild();
 }

build_runner_core/lib/src/environment/io_environment.dart

@@ -8,6 +8,7 @@ import 'dart:io';
 import 'package:build/build.dart';
 import 'package:logging/logging.dart';
 
+import '../asset/batch.dart';
 import '../asset/file_based.dart';
 import '../asset/reader.dart';
 import '../asset/writer.dart';
@@ -34,17 +35,33 @@ class IOEnvironment implements BuildEnvironment {
 
   final PackageGraph _packageGraph;
 
-  IOEnvironment(this._packageGraph,
-      {bool? assumeTty, bool outputSymlinksOnly = false})
-      : _isInteractive = assumeTty == true || _canPrompt(),
-        _outputSymlinksOnly = outputSymlinksOnly,
-        reader = FileBasedAssetReader(_packageGraph),
-        writer = FileBasedAssetWriter(_packageGraph) {
-    if (_outputSymlinksOnly && Platform.isWindows) {
+  IOEnvironment._(this.reader, this.writer, this._isInteractive,
+      this._outputSymlinksOnly, this._packageGraph);
+
+  factory IOEnvironment(
+    PackageGraph packageGraph, {
+    bool? assumeTty,
+    bool outputSymlinksOnly = false,
+    bool lowResourcesMode = false,
+  }) {
+    if (outputSymlinksOnly && Platform.isWindows) {
       _logger.warning('Symlinks to files are not yet working on Windows, you '
           'may experience issues using this mode. Follow '
           'https://github.com/dart-lang/sdk/issues/33966 for updates.');
     }
+
+    var fileReader = FileBasedAssetReader(packageGraph);
+    var fileWriter = FileBasedAssetWriter(packageGraph);
+
+    var (reader, writer) = lowResourcesMode
+        ? (fileReader, fileWriter)
+        : wrapInBatch(
+            reader: fileReader,
+            pathProvidingReader: fileReader,
+            writer: fileWriter);
+
+    return IOEnvironment._(reader, writer, assumeTty == true || _canPrompt(),
+        outputSymlinksOnly, packageGraph);
   }
 
   @override

build_runner_core/lib/src/generate/build_impl.dart

@@ -93,7 +93,12 @@ class BuildImpl {
       Set<BuildFilter> buildFilters = const {}}) {
     finalizedReader.reset(_buildPaths(buildDirs), buildFilters);
     return _SingleBuild(this, buildDirs, buildFilters).run(updates)
-      ..whenComplete(_resolvers.reset);
+      ..whenComplete(_handleCompletedBuild);
+  }
+
+  Future<void> _handleCompletedBuild() async {
+    await _environment.writer.completeBuild();
+    _resolvers.reset();
   }
 
   static Future<BuildImpl> create(

build_runner_core/pubspec.yaml

@@ -1,5 +1,5 @@
 name: build_runner_core
-version: 7.3.3-wip
+version: 8.0.0-wip
 description: Core tools to organize the structure of a build and run Builders.
 repository: https://github.com/dart-lang/build/tree/master/build_runner_core
 
@@ -49,3 +49,7 @@ dev_dependencies:
 
 topics:
  - build-runner
+
+dependency_overrides:
+  build_runner:
+    path: ../build_runner