Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[native_assets_builder] Use package:files FileSystem abstraction #1825

Merged
merged 2 commits into from
Dec 19, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pkgs/native_assets_builder/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
- Various fixes to caching.
- **Breaking change** `BuildConfig.targetOS` is now only provided if
`buildAssetTypes` contains the code asset.
- **Breaking change** `NativeAssetsBuildRunner` and `PackageLayout` now take a
`FileSystem` from `package:file/file.dart`s.

## 0.9.0

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
import 'dart:io' show Process;

import 'package:graphs/graphs.dart' as graphs;
import 'package:logging/logging.dart';
Expand Down
56 changes: 31 additions & 25 deletions pkgs/native_assets_builder/lib/src/build_runner/build_runner.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@

import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:io' show Platform;

import 'package:file/file.dart';
import 'package:logging/logging.dart';
import 'package:native_assets_cli/native_assets_cli_internal.dart';
import 'package:package_config/package_config.dart';
Expand Down Expand Up @@ -63,15 +64,18 @@ typedef ApplicationAssetValidator = Future<ValidationErrors> Function(
/// [BuildConfig] and [LinkConfig]! For more info see:
/// https://github.com/dart-lang/native/issues/1319
class NativeAssetsBuildRunner {
final FileSystem _fileSystem;
final Logger logger;
final Uri dartExecutable;
final Duration singleHookTimeout;

NativeAssetsBuildRunner({
required this.logger,
required this.dartExecutable,
required FileSystem fileSystem,
Duration? singleHookTimeout,
}) : singleHookTimeout = singleHookTimeout ?? const Duration(minutes: 5);
}) : _fileSystem = fileSystem,
singleHookTimeout = singleHookTimeout ?? const Duration(minutes: 5);

/// [workingDirectory] is expected to contain `.dart_tool`.
///
Expand All @@ -98,7 +102,8 @@ class NativeAssetsBuildRunner {
required List<String> buildAssetTypes,
required bool linkingEnabled,
}) async {
packageLayout ??= await PackageLayout.fromRootPackageRoot(workingDirectory);
packageLayout ??=
await PackageLayout.fromRootPackageRoot(_fileSystem, workingDirectory);

final (buildPlan, packageGraph) = await _makePlan(
hook: Hook.build,
Expand Down Expand Up @@ -203,7 +208,8 @@ class NativeAssetsBuildRunner {
required List<String> buildAssetTypes,
required BuildResult buildResult,
}) async {
packageLayout ??= await PackageLayout.fromRootPackageRoot(workingDirectory);
packageLayout ??=
await PackageLayout.fromRootPackageRoot(_fileSystem, workingDirectory);

final (buildPlan, packageGraph) = await _makePlan(
hook: Hook.link,
Expand Down Expand Up @@ -231,9 +237,9 @@ class NativeAssetsBuildRunner {

File? resourcesFile;
if (resourceIdentifiers != null) {
resourcesFile = File.fromUri(buildDirUri.resolve('resources.json'));
resourcesFile = _fileSystem.file(buildDirUri.resolve('resources.json'));
await resourcesFile.create();
await File.fromUri(resourceIdentifiers).copy(resourcesFile.path);
await _fileSystem.file(resourceIdentifiers).copy(resourcesFile.path);
}
configBuilder.setupLinkRunConfig(
outputDirectory: outDirUri,
Expand Down Expand Up @@ -291,14 +297,14 @@ class NativeAssetsBuildRunner {
final buildDirUri =
packageLayout.dartToolNativeAssetsBuilder.resolve('$buildDirName/');
final outDirUri = buildDirUri.resolve('out/');
final outDir = Directory.fromUri(outDirUri);
final outDir = _fileSystem.directory(outDirUri);
if (!await outDir.exists()) {
// TODO(https://dartbug.com/50565): Purge old or unused folders.
await outDir.create(recursive: true);
}
final outDirSharedUri = packageLayout.dartToolNativeAssetsBuilder
.resolve('shared/${package.name}/$hook/');
final outDirShared = Directory.fromUri(outDirSharedUri);
final outDirShared = _fileSystem.directory(outDirSharedUri);
if (!await outDirShared.exists()) {
// TODO(https://dartbug.com/50565): Purge old or unused folders.
await outDirShared.create(recursive: true);
Expand All @@ -318,9 +324,10 @@ class NativeAssetsBuildRunner {
final environment = _filteredEnvironment(_environmentVariablesFilter);
final outDir = config.outputDirectory;
return await runUnderDirectoriesLock(
_fileSystem,
[
Directory.fromUri(config.outputDirectoryShared.parent),
Directory.fromUri(config.outputDirectory.parent),
_fileSystem.directory(config.outputDirectoryShared).parent,
_fileSystem.directory(config.outputDirectory).parent,
],
timeout: singleHookTimeout,
logger: logger,
Expand All @@ -338,13 +345,13 @@ class NativeAssetsBuildRunner {
final (hookKernelFile, hookHashes) = hookCompileResult;

final buildOutputFile =
File.fromUri(config.outputDirectory.resolve(hook.outputName));
final dependenciesHashFile = File.fromUri(
_fileSystem.file(config.outputDirectory.resolve(hook.outputName));
final dependenciesHashFile = _fileSystem.file(
config.outputDirectory
.resolve('../dependencies.dependencies_hash_file.json'),
);
final dependenciesHashes =
DependenciesHashFile(file: dependenciesHashFile);
DependenciesHashFile(_fileSystem, file: dependenciesHashFile);
final lastModifiedCutoffTime = DateTime.now();
if (buildOutputFile.existsSync() && dependenciesHashFile.existsSync()) {
late final HookOutput output;
Expand Down Expand Up @@ -446,9 +453,9 @@ ${e.message}
final configFileContents =
const JsonEncoder.withIndent(' ').convert(config.json);
logger.info('config.json contents: $configFileContents');
await File.fromUri(configFile).writeAsString(configFileContents);
await _fileSystem.file(configFile).writeAsString(configFileContents);
final hookOutputUri = config.outputDirectory.resolve(hook.outputName);
final hookOutputFile = File.fromUri(hookOutputUri);
final hookOutputFile = _fileSystem.file(hookOutputUri);
if (await hookOutputFile.exists()) {
// Ensure we'll never read outdated build results.
await hookOutputFile.delete();
Expand All @@ -472,7 +479,8 @@ ${e.message}
var deleteOutputIfExists = false;
try {
if (result.exitCode != 0) {
final printWorkingDir = workingDirectory != Directory.current.uri;
final printWorkingDir =
workingDirectory != _fileSystem.currentDirectory.uri;
final commandString = [
if (printWorkingDir) '(cd ${workingDirectory.toFilePath()};',
dartExecutable.toFilePath(),
Expand Down Expand Up @@ -558,16 +566,17 @@ ${e.message}
) async {
// Don't invalidate cache with environment changes.
final environmentForCaching = <String, String>{};
final kernelFile = File.fromUri(
final kernelFile = _fileSystem.file(
outputDirectory.resolve('../hook.dill'),
);
final depFile = File.fromUri(
final depFile = _fileSystem.file(
outputDirectory.resolve('../hook.dill.d'),
);
final dependenciesHashFile = File.fromUri(
final dependenciesHashFile = _fileSystem.file(
outputDirectory.resolve('../hook.dependencies_hash_file.json'),
);
final dependenciesHashes = DependenciesHashFile(file: dependenciesHashFile);
final dependenciesHashes =
DependenciesHashFile(_fileSystem, file: dependenciesHashFile);
final lastModifiedCutoffTime = DateTime.now();
var mustCompile = false;
if (!await dependenciesHashFile.exists()) {
Expand Down Expand Up @@ -641,7 +650,8 @@ ${e.message}
);
var success = true;
if (compileResult.exitCode != 0) {
final printWorkingDir = workingDirectory != Directory.current.uri;
final printWorkingDir =
workingDirectory != _fileSystem.currentDirectory.uri;
final commandString = [
if (printWorkingDir) '(cd ${workingDirectory.toFilePath()};',
dartExecutable.toFilePath(),
Expand Down Expand Up @@ -780,10 +790,6 @@ ${compileResult.stdout}
}
}

extension on Uri {
Uri get parent => File(toFilePath()).parent.uri;
}

/// Parses depfile contents.
///
/// Format: `path/to/my.dill: path/to/my.dart, path/to/more.dart`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,22 @@
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
import 'dart:io' show Platform;
import 'dart:typed_data';

import 'package:crypto/crypto.dart';
import 'package:file/file.dart';

import '../utils/file.dart';
import '../utils/uri.dart';

class DependenciesHashFile {
DependenciesHashFile({
DependenciesHashFile(
this._fileSystem, {
mkustermann marked this conversation as resolved.
Show resolved Hide resolved
required this.file,
});

final FileSystem _fileSystem;
final File file;
FileSystemHashes _hashes = FileSystemHashes();

Expand Down Expand Up @@ -51,7 +54,7 @@ class DependenciesHashFile {
Uri? modifiedAfterTimeStamp;
for (final uri in fileSystemEntities) {
int hash;
if ((await uri.fileSystemEntity.lastModified())
if ((await _fileSystem.fileSystemEntity(uri).lastModified(_fileSystem))
.isAfter(fileSystemValidBeforeLastModified)) {
hash = _hashLastModifiedAfterCutoff;
modifiedAfterTimeStamp = uri;
Expand Down Expand Up @@ -124,15 +127,15 @@ class DependenciesHashFile {
}

Future<int> _hashFile(Uri uri) async {
final file = File.fromUri(uri);
final file = _fileSystem.file(uri);
if (!await file.exists()) {
return _hashNotExists;
}
return _md5int64(await file.readAsBytes());
}

Future<int> _hashDirectory(Uri uri) async {
final directory = Directory.fromUri(uri);
final directory = _fileSystem.directory(uri);
if (!await directory.exists()) {
return _hashNotExists;
}
Expand Down
13 changes: 9 additions & 4 deletions pkgs/native_assets_builder/lib/src/locking/locking.dart
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:io';
import 'dart:io' show Platform, pid;
import 'package:file/file.dart';

import 'package:logging/logging.dart';

Future<T> runUnderDirectoriesLock<T>(
FileSystem fileSystem,
List<Directory> directories,
Future<T> Function() callback, {
Duration? timeout,
Expand All @@ -17,8 +19,10 @@ Future<T> runUnderDirectoriesLock<T>(
return await callback();
}
return await runUnderDirectoryLock(
fileSystem,
directories.first,
mkustermann marked this conversation as resolved.
Show resolved Hide resolved
() => runUnderDirectoriesLock<T>(
fileSystem,
directories.skip(1).toList(),
callback,
timeout: timeout,
Expand All @@ -41,13 +45,14 @@ Future<T> runUnderDirectoriesLock<T>(
/// If provided, the [logger] streams information on the the locking status, and
/// also streams error messages.
Future<T> runUnderDirectoryLock<T>(
FileSystem fileSystem,
Directory directory,
Future<T> Function() callback, {
Duration? timeout,
Logger? logger,
}) async {
const lockFileName = '.lock';
final lockFile = _fileInDir(directory, lockFileName);
final lockFile = _fileInDir(fileSystem, directory, lockFileName);
return _runUnderFileLock(
lockFile,
callback,
Expand All @@ -56,11 +61,11 @@ Future<T> runUnderDirectoryLock<T>(
);
}

File _fileInDir(Directory path, String filename) {
File _fileInDir(FileSystem fileSystem, Directory path, String filename) {
mkustermann marked this conversation as resolved.
Show resolved Hide resolved
final dirPath = path.path;
var separator = Platform.pathSeparator;
if (dirPath.endsWith(separator)) separator = '';
return File('$dirPath$separator$filename');
return fileSystem.file('$dirPath$separator$filename');
}

/// Run [callback] with this Dart process having exclusive access to [file].
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:io';

import 'package:file/file.dart';
import 'package:native_assets_cli/native_assets_cli_internal.dart';
import 'package:package_config/package_config.dart';

Expand All @@ -15,6 +14,8 @@ import 'package:package_config/package_config.dart';
/// The directory layout follows pub's convention for caching:
/// https://dart.dev/tools/pub/package-layout#project-specific-caching-for-tools
class PackageLayout {
final FileSystem _fileSystem;

/// The root folder of the current dart invocation root package.
///
/// `$rootPackageRoot`.
Expand All @@ -29,26 +30,30 @@ class PackageLayout {

final Uri packageConfigUri;

PackageLayout._(
this.rootPackageRoot, this.packageConfig, this.packageConfigUri);
PackageLayout._(this._fileSystem, this.rootPackageRoot, this.packageConfig,
mkustermann marked this conversation as resolved.
Show resolved Hide resolved
this.packageConfigUri);

factory PackageLayout.fromPackageConfig(
FileSystem fileSystem,
PackageConfig packageConfig,
Uri packageConfigUri,
mkustermann marked this conversation as resolved.
Show resolved Hide resolved
) {
assert(File.fromUri(packageConfigUri).existsSync());
assert(fileSystem.file(packageConfigUri).existsSync());
packageConfigUri = packageConfigUri.normalizePath();
final rootPackageRoot = packageConfigUri.resolve('../');
return PackageLayout._(rootPackageRoot, packageConfig, packageConfigUri);
return PackageLayout._(
fileSystem, rootPackageRoot, packageConfig, packageConfigUri);
mkustermann marked this conversation as resolved.
Show resolved Hide resolved
}

static Future<PackageLayout> fromRootPackageRoot(Uri rootPackageRoot) async {
static Future<PackageLayout> fromRootPackageRoot(
FileSystem fileSystem, Uri rootPackageRoot) async {
mkustermann marked this conversation as resolved.
Show resolved Hide resolved
rootPackageRoot = rootPackageRoot.normalizePath();
final packageConfigUri =
rootPackageRoot.resolve('.dart_tool/package_config.json');
assert(await File.fromUri(packageConfigUri).exists());
assert(await fileSystem.file(packageConfigUri).exists());
final packageConfig = await loadPackageConfigUri(packageConfigUri);
return PackageLayout._(rootPackageRoot, packageConfig, packageConfigUri);
return PackageLayout._(
fileSystem, rootPackageRoot, packageConfig, packageConfigUri);
}

/// The .dart_tool directory is used to store built artifacts and caches.
Expand Down Expand Up @@ -107,12 +112,12 @@ class PackageLayout {
for (final package in packageConfig.packages) {
final packageRoot = package.root;
if (packageRoot.scheme == 'file') {
if (await File.fromUri(
packageRoot.resolve('hook/').resolve(hook.scriptName),
).exists() ||
await File.fromUri(
packageRoot.resolve(hook.scriptName),
).exists()) {
if (await _fileSystem
.file(packageRoot.resolve('hook/').resolve(hook.scriptName))
.exists() ||
await _fileSystem
.file(packageRoot.resolve(hook.scriptName))
.exists()) {
result.add(package);
}
}
Expand Down
Loading
Loading