diff --git a/pkg/dartdev/lib/src/commands/build.dart b/pkg/dartdev/lib/src/commands/build.dart index a7784e9f3b36..f0dd3890e31f 100644 --- a/pkg/dartdev/lib/src/commands/build.dart +++ b/pkg/dartdev/lib/src/commands/build.dart @@ -8,6 +8,7 @@ import 'dart:io'; import 'package:dart2native/generate.dart'; import 'package:dartdev/src/commands/compile.dart'; import 'package:dartdev/src/experiments.dart'; +import 'package:dartdev/src/native_assets_bundling.dart'; import 'package:dartdev/src/sdk.dart'; import 'package:dartdev/src/utils.dart'; import 'package:front_end/src/api_prototype/compiler_options.dart' @@ -21,9 +22,6 @@ import 'package:vm/target_os.dart'; // For possible --target-os values. import '../core.dart'; import '../native_assets.dart'; -const _libOutputDirectory = 'lib'; -const _dataOutputDirectory = 'assets'; - class BuildCommand extends DartdevCommand { static const String cmdName = 'build'; static const String outputOptionName = 'output'; @@ -195,8 +193,6 @@ class BuildCommand extends DartdevCommand { return 255; } - final tempUri = tempDir.uri; - Uri? assetsDartUri; final allAssets = [...buildResult.assets, ...linkResult.assets]; final staticAssets = allAssets .whereType() @@ -207,22 +203,25 @@ class BuildCommand extends DartdevCommand { Use linkMode as dynamic library instead."""); return 255; } + + Uri? nativeAssetsYamlUri; if (allAssets.isNotEmpty) { - final targetMapping = _targetMapping(allAssets, target); - assetsDartUri = await _writeAssetsYaml( - targetMapping.map((e) => e.target).toList(), - assetsDartUri, - tempUri, + stdout.writeln( + 'Bundling ${allAssets.length} built assets: ' + '${allAssets.map((e) => e.id).join(', ')}.', ); - if (allAssets.isNotEmpty) { - stdout.writeln( - 'Copying ${allAssets.length} build assets: ${allAssets.map((e) => e.id)}'); - _copyAssets(targetMapping, outputUri); - } + final kernelAssets = await bundleNativeAssets( + allAssets, + target, + outputUri, + relocatable: true, + ); + nativeAssetsYamlUri = + await writeNativeAssetsYaml(kernelAssets, tempDir.uri); } await snapshotGenerator.generate( - nativeAssets: assetsDartUri?.toFilePath(), + nativeAssets: nativeAssetsYamlUri?.toFilePath(), ); // End linking here. @@ -231,42 +230,6 @@ Use linkMode as dynamic library instead."""); } return 0; } - - List<({AssetImpl asset, KernelAsset target})> _targetMapping( - Iterable assets, - Target target, - ) { - return [ - for (final asset in assets) - (asset: asset, target: asset.targetLocation(target)), - ]; - } - - void _copyAssets( - List<({AssetImpl asset, KernelAsset target})> assetTargetLocations, - Uri output, - ) { - for (final (asset: asset, target: target) in assetTargetLocations) { - final targetPath = target.path; - if (targetPath is KernelAssetRelativePath) { - asset.file!.copyTo(targetPath, output); - } - } - } - - Future _writeAssetsYaml( - List assetTargetLocations, - Uri? nativeAssetsDartUri, - Uri tempUri, - ) async { - stdout.writeln('Writing native_assets.yaml.'); - nativeAssetsDartUri = tempUri.resolve('native_assets.yaml'); - final assetsContent = - KernelAssets(assetTargetLocations).toNativeAssetsFile(); - await Directory.fromUri(nativeAssetsDartUri.resolve('.')).create(); - await File(nativeAssetsDartUri.toFilePath()).writeAsString(assetsContent); - return nativeAssetsDartUri; - } } extension on String { @@ -275,67 +238,6 @@ extension on String { String removeDotDart() => replaceFirst(RegExp(r'\.dart$'), ''); } -extension on Uri { - void copyTo(KernelAssetRelativePath target, Uri outputUri) { - if (this != target.uri) { - final targetUri = outputUri.resolveUri(target.uri); - File.fromUri(targetUri).createSync( - recursive: true, - exclusive: true, - ); - File.fromUri(this).copySync(targetUri.toFilePath()); - } - } -} - -extension on AssetImpl { - KernelAsset targetLocation(Target target) { - return switch (this) { - NativeCodeAssetImpl nativeAsset => nativeAsset.targetLocation(target), - DataAssetImpl dataAsset => dataAsset.targetLocation(target), - AssetImpl() => throw UnimplementedError(), - }; - } -} - -extension on NativeCodeAssetImpl { - KernelAsset targetLocation(Target target) { - final KernelAssetPath kernelAssetPath; - switch (linkMode) { - case DynamicLoadingSystemImpl dynamicLoading: - kernelAssetPath = KernelAssetSystemPath(dynamicLoading.uri); - case LookupInExecutableImpl _: - kernelAssetPath = KernelAssetInExecutable(); - case LookupInProcessImpl _: - kernelAssetPath = KernelAssetInProcess(); - case DynamicLoadingBundledImpl _: - kernelAssetPath = KernelAssetRelativePath( - Uri(path: path.join(_libOutputDirectory, file!.pathSegments.last)), - ); - default: - throw Exception( - 'Unsupported NativeCodeAsset linkMode ${linkMode.runtimeType} in asset $this', - ); - } - return KernelAsset( - id: id, - target: target, - path: kernelAssetPath, - ); - } -} - -extension on DataAssetImpl { - KernelAsset targetLocation(Target target) { - return KernelAsset( - id: id, - target: target, - path: KernelAssetRelativePath( - Uri(path: path.join(_dataOutputDirectory, file.pathSegments.last))), - ); - } -} - // TODO(https://github.com/dart-lang/package_config/issues/126): Expose this // logic in package:package_config. Future packageConfigUri(Uri uri) async { diff --git a/pkg/dartdev/lib/src/native_assets.dart b/pkg/dartdev/lib/src/native_assets.dart index b21cfc3c0b19..b68a08869972 100644 --- a/pkg/dartdev/lib/src/native_assets.dart +++ b/pkg/dartdev/lib/src/native_assets.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'dart:io'; +import 'package:dartdev/src/native_assets_bundling.dart'; import 'package:dartdev/src/sdk.dart'; import 'package:dartdev/src/utils.dart'; import 'package:logging/logging.dart'; @@ -79,57 +80,27 @@ Future<(bool success, Uri? nativeAssetsYaml)> compileNativeAssetsJitYamlFile({ if (!success) { return (false, null); } - final kernelAssets = KernelAssets([ - ...[ - for (final asset in assets.whereType()) - _targetLocation(asset), - ], - ...[ - for (final asset in assets.whereType()) - _dataTargetLocation(asset), - ] - ]); - final workingDirectory = Directory.current.uri; - final assetsUri = workingDirectory.resolve('.dart_tool/native_assets.yaml'); - final nativeAssetsYaml = '''# Native assets mapping for host OS in JIT mode. -# Generated by dartdev and package:native_assets_builder. -${kernelAssets.toNativeAssetsFile()}'''; - final assetFile = File(assetsUri.toFilePath()); - await assetFile.writeAsString(nativeAssetsYaml); - return (true, assetsUri); -} + final dartToolUri = Directory.current.uri.resolve('.dart_tool/'); + final outputUri = dartToolUri.resolve('native_assets/'); + await Directory.fromUri(outputUri).create(recursive: true); -KernelAsset _targetLocation(NativeCodeAssetImpl asset) { - final linkMode = asset.linkMode; - final KernelAssetPath kernelAssetPath; - switch (linkMode) { - case DynamicLoadingSystemImpl _: - kernelAssetPath = KernelAssetSystemPath(linkMode.uri); - case LookupInExecutableImpl _: - kernelAssetPath = KernelAssetInExecutable(); - case LookupInProcessImpl _: - kernelAssetPath = KernelAssetInProcess(); - case DynamicLoadingBundledImpl _: - kernelAssetPath = KernelAssetAbsolutePath(asset.file!); - default: - throw Exception( - 'Unsupported NativeCodeAsset linkMode ${linkMode.runtimeType} in asset $asset', - ); - } - return KernelAsset( - id: asset.id, - target: Target.fromArchitectureAndOS(asset.architecture!, asset.os), - path: kernelAssetPath, + final kernelAssets = await bundleNativeAssets( + assets, + Target.current, + outputUri, + relocatable: false, ); -} -KernelAsset _dataTargetLocation(DataAssetImpl asset) { - return KernelAsset( - id: asset.id, - target: Target.current, - path: KernelAssetAbsolutePath(asset.file), + final nativeAssetsYamlUri = await writeNativeAssetsYaml( + kernelAssets, + dartToolUri, + header: '''# Native assets mapping for host OS in JIT mode. +# Generated by dartdev and package:native_assets_builder. +''', ); + + return (true, nativeAssetsYamlUri); } Future warnOnNativeAssets() async { diff --git a/pkg/dartdev/lib/src/native_assets_bundling.dart b/pkg/dartdev/lib/src/native_assets_bundling.dart new file mode 100644 index 000000000000..5da6b0b58435 --- /dev/null +++ b/pkg/dartdev/lib/src/native_assets_bundling.dart @@ -0,0 +1,168 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:dartdev/src/native_assets_macos.dart'; +import 'package:native_assets_builder/native_assets_builder.dart'; +import 'package:native_assets_cli/native_assets_cli.dart'; +import 'package:native_assets_cli/native_assets_cli_internal.dart'; + +final libOutputDirectoryUri = Uri.file('lib/'); +final dataOutputDirectoryUri = Uri.file('assets/'); + +Future bundleNativeAssets( + Iterable assets, + Target target, + Uri outputUri, { + required bool relocatable, +}) async { + final targetMapping = _targetMapping(assets, target, outputUri, relocatable); + await _copyAssets(targetMapping, target, outputUri, relocatable); + return KernelAssets(targetMapping.map((asset) => asset.target).toList()); +} + +Future writeNativeAssetsYaml( + KernelAssets assets, + Uri outputUri, { + String? header, +}) async { + final nativeAssetsYamlUri = outputUri.resolve('native_assets.yaml'); + final nativeAssetsYamlFile = File(nativeAssetsYamlUri.toFilePath()); + await nativeAssetsYamlFile.create(recursive: true); + + var contents = assets.toNativeAssetsFile(); + if (header != null) { + contents = '$header\n$contents'; + } + + await nativeAssetsYamlFile.writeAsString(contents); + return nativeAssetsYamlUri; +} + +Future _copyAssets( + List<({Asset asset, KernelAsset target})> targetMapping, + Target target, + Uri outputUri, + bool relocatable, +) async { + var codeAssetsNeedCopying = false; + final codeAssetSourceToTargetUris = <({Uri source, Uri target})>[]; + + await Future.wait(targetMapping.map((entry) async { + final (:asset, :target) = entry; + final targetPath = target.path; + if (targetPath + case KernelAssetRelativePath(:final uri) || + KernelAssetAbsolutePath(:final uri)) { + final targetUri = outputUri.resolveUri(uri); + + switch (asset) { + case NativeCodeAsset(:final file): + codeAssetsNeedCopying = + codeAssetsNeedCopying || file!.needsCopying(targetUri); + codeAssetSourceToTargetUris.add((source: file!, target: targetUri)); + case DataAsset(:final file): + await file!.copyTo(targetUri); + default: + throw UnimplementedError(); + } + } + })); + + if (codeAssetsNeedCopying) { + await Future.wait( + codeAssetSourceToTargetUris + .map((entry) => entry.source.copyTo(entry.target)), + ); + + final targetUris = + codeAssetSourceToTargetUris.map((entry) => entry.target).toList(); + + if (target.os == OS.macOS) { + await rewriteInstallNames(targetUris, relocatable: relocatable); + } + } +} + +List<({Asset asset, KernelAsset target})> _targetMapping( + Iterable assets, + Target target, + Uri outputUri, + bool relocatable, +) { + return [ + for (final asset in assets) + ( + asset: asset, + target: asset.targetLocation(target, outputUri, relocatable) + ), + ]; +} + +extension on Asset { + KernelAsset targetLocation(Target target, Uri outputUri, bool relocatable) { + return switch (this) { + NativeCodeAsset nativeAsset => + nativeAsset.targetLocation(target, outputUri, relocatable), + DataAsset dataAsset => + dataAsset.targetLocation(target, outputUri, relocatable), + _ => throw UnimplementedError(), + }; + } +} + +extension on NativeCodeAsset { + KernelAsset targetLocation(Target target, Uri outputUri, bool relocatable) { + final kernelAssetPath = switch (linkMode) { + DynamicLoadingSystem(:final uri) => KernelAssetSystemPath(uri), + LookupInExecutable() => KernelAssetInExecutable(), + LookupInProcess() => KernelAssetInProcess(), + DynamicLoadingBundled() => () { + final relativeUri = + libOutputDirectoryUri.resolve(file!.pathSegments.last); + return relocatable + ? KernelAssetRelativePath(relativeUri) + : KernelAssetAbsolutePath(outputUri.resolveUri(relativeUri)); + }(), + _ => throw Exception( + 'Unsupported NativeCodeAsset linkMode ${linkMode.runtimeType} in asset $this', + ), + }; + return KernelAsset( + id: id, + target: target, + path: kernelAssetPath, + ); + } +} + +extension on DataAsset { + KernelAsset targetLocation(Target target, Uri outputUri, bool relocatable) { + final relativeUri = dataOutputDirectoryUri.resolve(file!.pathSegments.last); + return KernelAsset( + id: id, + target: target, + path: relocatable + ? KernelAssetRelativePath(relativeUri) + : KernelAssetAbsolutePath(outputUri.resolveUri(relativeUri)), + ); + } +} + +extension on Uri { + bool needsCopying(Uri targetUri) { + final targetFile = File.fromUri(targetUri); + return targetFile.existsSync() + ? targetFile + .lastModifiedSync() + .isBefore(File.fromUri(this).lastModifiedSync()) + : true; + } + + Future copyTo(Uri targetUri) async{ + await File.fromUri(targetUri).create(recursive: true); + await File.fromUri(this).copy(targetUri.toFilePath()); + } +} diff --git a/pkg/dartdev/lib/src/native_assets_macos.dart b/pkg/dartdev/lib/src/native_assets_macos.dart new file mode 100644 index 000000000000..edb69b9b0b7d --- /dev/null +++ b/pkg/dartdev/lib/src/native_assets_macos.dart @@ -0,0 +1,160 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:dartdev/src/native_assets_bundling.dart'; +import 'package:native_assets_cli/native_assets_cli.dart'; + +final _rpathUri = Uri.file('@rpath/'); + +Future rewriteInstallNames( + List dylibs, { + required bool relocatable, +}) async { + final oldToNewInstallNames = {}; + final dylibInfos = <(Uri, String)>[]; + + await Future.wait(dylibs.map((dylib) async { + final newInstallName = relocatable + ? _rpathUri + .resolveUri(libOutputDirectoryUri) + .resolve(dylib.pathSegments.last) + .toFilePath() + : dylib.toFilePath(); + final oldInstallName = await _getInstallName(dylib); + oldToNewInstallNames[oldInstallName] = newInstallName; + dylibInfos.add((dylib, newInstallName)); + })); + + await Future.wait(dylibInfos.map((info) async { + final (dylib, newInstallName) = info; + await _setInstallNames(dylib, newInstallName, oldToNewInstallNames); + await _codeSignDylib(dylib); + })); +} + +Future _getInstallName(Uri dylib) async { + final otoolResult = await Process.run( + 'otool', + [ + '-D', + dylib.toFilePath(), + ], + ); + if (otoolResult.exitCode != 0) { + throw Exception( + 'Failed to get install name for dylib $dylib: ${otoolResult.stderr}', + ); + } + final architectureSections = + parseOtoolArchitectureSections(otoolResult.stdout); + if (architectureSections.length != 1) { + throw Exception( + 'Expected a single architecture section in otool output: $otoolResult', + ); + } + return architectureSections.values.first.single; +} + +Future _setInstallNames( + Uri dylib, + String newInstallName, + Map oldToNewInstallNames, +) async { + final installNameToolResult = await Process.run( + 'install_name_tool', + [ + '-id', + newInstallName, + for (final entry in oldToNewInstallNames.entries) ...[ + '-change', + entry.key, + entry.value, + ], + dylib.toFilePath(), + ], + ); + if (installNameToolResult.exitCode != 0) { + throw Exception( + 'Failed to set install names for dylib $dylib:\n' + 'id -> $newInstallName\n' + 'dependencies -> $oldToNewInstallNames\n' + '${installNameToolResult.stderr}', + ); + } +} + +Future _codeSignDylib(Uri dylib) async { + final codesignResult = await Process.run( + 'codesign', + [ + '--force', + '--sign', + '-', + dylib.toFilePath(), + ], + ); + if (codesignResult.exitCode != 0) { + throw Exception( + 'Failed to codesign dylib $dylib: ${codesignResult.stderr}', + ); + } +} + +Map> parseOtoolArchitectureSections(String output) { + // The output of `otool -D`, for example, looks like below. For each + // architecture, there is a separate section. + // + // /build/native_assets/ios/buz.framework/buz (architecture x86_64): + // @rpath/libbuz.dylib + // /build/native_assets/ios/buz.framework/buz (architecture arm64): + // @rpath/libbuz.dylib + // + // Some versions of `otool` don't print the architecture name if the + // binary only has one architecture: + // + // /build/native_assets/ios/buz.framework/buz: + // @rpath/libbuz.dylib + + const Map outputArchitectures = { + 'arm': Architecture.arm, + 'arm64': Architecture.arm64, + 'x86_64': Architecture.x64, + }; + final RegExp architectureHeaderPattern = + RegExp(r'^[^(]+( \(architecture (.+)\))?:$'); + final Iterator lines = output.trim().split('\n').iterator; + Architecture? currentArchitecture; + final Map> architectureSections = + >{}; + + while (lines.moveNext()) { + final String line = lines.current; + final Match? architectureHeader = + architectureHeaderPattern.firstMatch(line); + if (architectureHeader != null) { + if (architectureSections.containsKey(null)) { + throw Exception( + 'Expected a single architecture section in otool output: $output', + ); + } + final String? architectureString = architectureHeader.group(2); + if (architectureString != null) { + currentArchitecture = outputArchitectures[architectureString]; + if (currentArchitecture == null) { + throw Exception( + 'Unknown architecture in otool output: $architectureString', + ); + } + } + architectureSections[currentArchitecture] = []; + continue; + } else { + architectureSections[currentArchitecture]!.add(line.trim()); + } + } + + return architectureSections; +} diff --git a/pkg/dartdev/test/native_assets/build_test.dart b/pkg/dartdev/test/native_assets/build_test.dart index e2ac0ce780ce..5b8575eedd15 100644 --- a/pkg/dartdev/test/native_assets/build_test.dart +++ b/pkg/dartdev/test/native_assets/build_test.dart @@ -6,6 +6,7 @@ import 'dart:io'; +import 'package:native_assets_cli/native_assets_cli.dart'; import 'package:native_assets_cli/native_assets_cli_internal.dart'; import 'package:test/test.dart'; @@ -189,6 +190,46 @@ void main(List args) { ); }); }); + + test( + 'dart build with native dynamic linking', + timeout: longTimeout, + () async { + await nativeAssetsTest('native_dynamic_linking', (packageUri) async { + await runDart( + arguments: [ + '--enable-experiment=native-assets', + 'build', + 'bin/native_dynamic_linking.dart', + ], + workingDirectory: packageUri, + logger: logger, + ); + + final outputDirectory = + Directory.fromUri(packageUri.resolve('bin/native_dynamic_linking')); + expect(outputDirectory.existsSync(), true); + + File dylibFile(String name) { + final libDirectoryUri = (outputDirectory.uri.resolve('lib/')); + final dylibBasename = + OS.current.libraryFileName(name, DynamicLoadingBundledImpl()); + return File.fromUri(libDirectoryUri.resolve(dylibBasename)); + } + + expect(dylibFile('add').existsSync(), true); + expect(dylibFile('math').existsSync(), true); + expect(dylibFile('debug').existsSync(), true); + + final proccessResult = await runProcess( + executable: outputDirectory.uri.resolve('native_dynamic_linking.exe'), + logger: logger, + throwOnUnexpectedExitCode: true, + ); + expect(proccessResult.stdout, contains('42')); + }); + }, + ); } Future _withTempDir(Future Function(Uri tempUri) fun) async { diff --git a/pkg/dartdev/test/native_assets/helpers.dart b/pkg/dartdev/test/native_assets/helpers.dart index 65264f5c88c4..5e1cb9363dcd 100644 --- a/pkg/dartdev/test/native_assets/helpers.dart +++ b/pkg/dartdev/test/native_assets/helpers.dart @@ -146,11 +146,26 @@ void expectDartAppStdout(String stdout) { } /// Logger that outputs the full trace when a test fails. -final logger = Logger('') - ..level = Level.ALL - ..onRecord.listen((record) { - printOnFailure('${record.level.name}: ${record.time}: ${record.message}'); - }); +Logger get logger => _logger ??= () { + // A new logger is lazily created for each test so that the messages + // captured by printOnFailure are scoped to the correct test. + addTearDown(() => _logger = null); + return _createTestLogger(); + }(); + +Logger? _logger; + +Logger createCapturingLogger(List capturedMessages) => + _createTestLogger(capturedMessages: capturedMessages); + +Logger _createTestLogger({List? capturedMessages}) => + Logger.detached('') + ..level = Level.ALL + ..onRecord.listen((record) { + printOnFailure( + '${record.level.name}: ${record.time}: ${record.message}'); + capturedMessages?.add(record.message); + }); final dartExecutable = Uri.file(Platform.resolvedExecutable); @@ -163,6 +178,7 @@ Future nativeAssetsTest( 'native_add', 'drop_dylib_link', 'add_asset_link', + 'native_dynamic_linking', ].contains(packageUnderTest)); return await inTempDir((tempUri) async { await copyTestProjects(tempUri, logger); diff --git a/pkg/dartdev/test/native_assets/run_test.dart b/pkg/dartdev/test/native_assets/run_test.dart index fb043c45372b..188766813b12 100644 --- a/pkg/dartdev/test/native_assets/run_test.dart +++ b/pkg/dartdev/test/native_assets/run_test.dart @@ -154,4 +154,19 @@ Couldn't resolve native function 'multiply' in 'package:drop_dylib_link/dylib_mu ); }); }); + + test('dart run with native dynamic linking', timeout: longTimeout, () async { + await nativeAssetsTest('native_dynamic_linking', (packageUri) async { + final result = await runDart( + arguments: [ + '--enable-experiment=native-assets', + 'run', + 'bin/native_dynamic_linking.dart', + ], + workingDirectory: packageUri, + logger: logger, + ); + expect(result.stdout, contains('42')); + }); + }); } diff --git a/pkg/dartdev/test/native_assets/test_test.dart b/pkg/dartdev/test/native_assets/test_test.dart index 7e4c0f5d827b..7d6cde5e3a0a 100644 --- a/pkg/dartdev/test/native_assets/test_test.dart +++ b/pkg/dartdev/test/native_assets/test_test.dart @@ -73,4 +73,26 @@ void main(List args) async { expect(result.stderr, contains('native_add')); }); }); + + test('dart test with native dynamic linking', timeout: longTimeout, () async { + await nativeAssetsTest('native_dynamic_linking', (packageUri) async { + final result = await runDart( + arguments: [ + '--enable-experiment=native-assets', + 'test', + ], + workingDirectory: packageUri, + logger: logger, + ); + expect( + result.stdout, + stringContainsInOrder( + [ + 'invoke native function', + 'All tests passed!', + ], + ), + ); + }); + }); }