Skip to content

Commit

Permalink
[native assets] Support dynamic linking of native code assets
Browse files Browse the repository at this point in the history 
The same bundling that is used for `dart build` is now also used for `dart test` and `dart run`, except that the output directory is `.dart_tool/native_assets`. This way all native code assets are placed next to each other in the `lib` directory, and loaded from there instead of loading them in place from where the build/link hooks placed them. By standardizing on this layout the different modes of running dart code that support native assets can use the same mechanisms to support dynamic linking between native code assets.

Also, on macOS install names of dylibs are rewritten to support dynamic linking, similar to the changes in flutter/flutter#153054.

Tests are added to verify that dynamic linking works as expected.

Related: dart-lang/native#190
Fixes: dart-lang#56459
  • Loading branch information
@blaugold
blaugold committed Aug 20, 2024
1 parent eb887c2 commit 91ff56e
Show file tree
Hide file tree
Showing 8 changed files with 459 additions and 164 deletions.
128 changes: 15 additions & 113 deletions pkg/dartdev/lib/src/commands/build.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import 'dart:io';
import 'package:dart2native/generate.dart';
import 'package:dartdev/src/commands/compile.dart';
import 'package:dartdev/src/experiments.dart';
import 'package:dartdev/src/native_assets_bundling.dart';
import 'package:dartdev/src/sdk.dart';
import 'package:dartdev/src/utils.dart';
import 'package:front_end/src/api_prototype/compiler_options.dart'
Expand All @@ -21,9 +22,6 @@ import 'package:vm/target_os.dart'; // For possible --target-os values.
import '../core.dart';
import '../native_assets.dart';

const _libOutputDirectory = 'lib';
const _dataOutputDirectory = 'assets';

class BuildCommand extends DartdevCommand {
static const String cmdName = 'build';
static const String outputOptionName = 'output';
Expand Down Expand Up @@ -195,8 +193,6 @@ class BuildCommand extends DartdevCommand {
return 255;
}

final tempUri = tempDir.uri;
Uri? assetsDartUri;
final allAssets = [...buildResult.assets, ...linkResult.assets];
final staticAssets = allAssets
.whereType<NativeCodeAssetImpl>()
Expand All @@ -207,22 +203,25 @@ class BuildCommand extends DartdevCommand {
Use linkMode as dynamic library instead.""");
return 255;
}

Uri? nativeAssetsYamlUri;
if (allAssets.isNotEmpty) {
final targetMapping = _targetMapping(allAssets, target);
assetsDartUri = await _writeAssetsYaml(
targetMapping.map((e) => e.target).toList(),
assetsDartUri,
tempUri,
stdout.writeln(
'Bundling ${allAssets.length} built assets: '
'${allAssets.map((e) => e.id).join(', ')}.',
);
if (allAssets.isNotEmpty) {
stdout.writeln(
'Copying ${allAssets.length} build assets: ${allAssets.map((e) => e.id)}');
_copyAssets(targetMapping, outputUri);
}
final kernelAssets = await bundleNativeAssets(
allAssets,
target,
outputUri,
relocatable: true,
);
nativeAssetsYamlUri =
await writeNativeAssetsYaml(kernelAssets, tempDir.uri);
}

await snapshotGenerator.generate(
nativeAssets: assetsDartUri?.toFilePath(),
nativeAssets: nativeAssetsYamlUri?.toFilePath(),
);

// End linking here.
Expand All @@ -231,42 +230,6 @@ Use linkMode as dynamic library instead.""");
}
return 0;
}

List<({AssetImpl asset, KernelAsset target})> _targetMapping(
Iterable<AssetImpl> assets,
Target target,
) {
return [
for (final asset in assets)
(asset: asset, target: asset.targetLocation(target)),
];
}

void _copyAssets(
List<({AssetImpl asset, KernelAsset target})> assetTargetLocations,
Uri output,
) {
for (final (asset: asset, target: target) in assetTargetLocations) {
final targetPath = target.path;
if (targetPath is KernelAssetRelativePath) {
asset.file!.copyTo(targetPath, output);
}
}
}

Future<Uri> _writeAssetsYaml(
List<KernelAsset> assetTargetLocations,
Uri? nativeAssetsDartUri,
Uri tempUri,
) async {
stdout.writeln('Writing native_assets.yaml.');
nativeAssetsDartUri = tempUri.resolve('native_assets.yaml');
final assetsContent =
KernelAssets(assetTargetLocations).toNativeAssetsFile();
await Directory.fromUri(nativeAssetsDartUri.resolve('.')).create();
await File(nativeAssetsDartUri.toFilePath()).writeAsString(assetsContent);
return nativeAssetsDartUri;
}
}

extension on String {
Expand All @@ -275,67 +238,6 @@ extension on String {
String removeDotDart() => replaceFirst(RegExp(r'\.dart$'), '');
}

extension on Uri {
void copyTo(KernelAssetRelativePath target, Uri outputUri) {
if (this != target.uri) {
final targetUri = outputUri.resolveUri(target.uri);
File.fromUri(targetUri).createSync(
recursive: true,
exclusive: true,
);
File.fromUri(this).copySync(targetUri.toFilePath());
}
}
}

extension on AssetImpl {
KernelAsset targetLocation(Target target) {
return switch (this) {
NativeCodeAssetImpl nativeAsset => nativeAsset.targetLocation(target),
DataAssetImpl dataAsset => dataAsset.targetLocation(target),
AssetImpl() => throw UnimplementedError(),
};
}
}

extension on NativeCodeAssetImpl {
KernelAsset targetLocation(Target target) {
final KernelAssetPath kernelAssetPath;
switch (linkMode) {
case DynamicLoadingSystemImpl dynamicLoading:
kernelAssetPath = KernelAssetSystemPath(dynamicLoading.uri);
case LookupInExecutableImpl _:
kernelAssetPath = KernelAssetInExecutable();
case LookupInProcessImpl _:
kernelAssetPath = KernelAssetInProcess();
case DynamicLoadingBundledImpl _:
kernelAssetPath = KernelAssetRelativePath(
Uri(path: path.join(_libOutputDirectory, file!.pathSegments.last)),
);
default:
throw Exception(
'Unsupported NativeCodeAsset linkMode ${linkMode.runtimeType} in asset $this',
);
}
return KernelAsset(
id: id,
target: target,
path: kernelAssetPath,
);
}
}

extension on DataAssetImpl {
KernelAsset targetLocation(Target target) {
return KernelAsset(
id: id,
target: target,
path: KernelAssetRelativePath(
Uri(path: path.join(_dataOutputDirectory, file.pathSegments.last))),
);
}
}

// TODO(https://github.com/dart-lang/package_config/issues/126): Expose this
// logic in package:package_config.
Future<Uri?> packageConfigUri(Uri uri) async {
Expand Down
63 changes: 17 additions & 46 deletions pkg/dartdev/lib/src/native_assets.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import 'dart:async';
import 'dart:io';

import 'package:dartdev/src/native_assets_bundling.dart';
import 'package:dartdev/src/sdk.dart';
import 'package:dartdev/src/utils.dart';
import 'package:logging/logging.dart';
Expand Down Expand Up @@ -79,57 +80,27 @@ Future<(bool success, Uri? nativeAssetsYaml)> compileNativeAssetsJitYamlFile({
if (!success) {
return (false, null);
}
final kernelAssets = KernelAssets([
...[
for (final asset in assets.whereType<NativeCodeAssetImpl>())
_targetLocation(asset),
],
...[
for (final asset in assets.whereType<DataAssetImpl>())
_dataTargetLocation(asset),
]
]);

final workingDirectory = Directory.current.uri;
final assetsUri = workingDirectory.resolve('.dart_tool/native_assets.yaml');
final nativeAssetsYaml = '''# Native assets mapping for host OS in JIT mode.
# Generated by dartdev and package:native_assets_builder.
${kernelAssets.toNativeAssetsFile()}''';
final assetFile = File(assetsUri.toFilePath());
await assetFile.writeAsString(nativeAssetsYaml);
return (true, assetsUri);
}
final dartToolUri = Directory.current.uri.resolve('.dart_tool/');
final outputUri = dartToolUri.resolve('native_assets/');
await Directory.fromUri(outputUri).create(recursive: true);

KernelAsset _targetLocation(NativeCodeAssetImpl asset) {
final linkMode = asset.linkMode;
final KernelAssetPath kernelAssetPath;
switch (linkMode) {
case DynamicLoadingSystemImpl _:
kernelAssetPath = KernelAssetSystemPath(linkMode.uri);
case LookupInExecutableImpl _:
kernelAssetPath = KernelAssetInExecutable();
case LookupInProcessImpl _:
kernelAssetPath = KernelAssetInProcess();
case DynamicLoadingBundledImpl _:
kernelAssetPath = KernelAssetAbsolutePath(asset.file!);
default:
throw Exception(
'Unsupported NativeCodeAsset linkMode ${linkMode.runtimeType} in asset $asset',
);
}
return KernelAsset(
id: asset.id,
target: Target.fromArchitectureAndOS(asset.architecture!, asset.os),
path: kernelAssetPath,
final kernelAssets = await bundleNativeAssets(
assets,
Target.current,
outputUri,
relocatable: false,
);
}

KernelAsset _dataTargetLocation(DataAssetImpl asset) {
return KernelAsset(
id: asset.id,
target: Target.current,
path: KernelAssetAbsolutePath(asset.file),
final nativeAssetsYamlUri = await writeNativeAssetsYaml(
kernelAssets,
dartToolUri,
header: '''# Native assets mapping for host OS in JIT mode.
# Generated by dartdev and package:native_assets_builder.
''',
);

return (true, nativeAssetsYamlUri);
}

Future<bool> warnOnNativeAssets() async {
Expand Down
Loading

0 comments on commit 91ff56e

Please sign in to comment.