Skip to content

Commit

Permalink
Repl binding (#443)
Browse files Browse the repository at this point in the history
* Added a separate REPL syntax

* Syntax updated

* Update CompilerConstants.cs

* Create Script.cs

* Update Script.cs

* Update Script.cs

* Update Script.cs

* Added compiler flag

* Rename

* Shuffling code

* Fixed compilation

* More API

* Fixed test namespaces

* Fixed binder cache

* Added source script module

* Parser fixes

* Update SourceScriptModuleSymbol.cs

* Added synthetized method

* Moved

* Better stuff

* Docs

* Update ScriptEvalFunctionSymbol.cs

* Additional structures

* Added bases for syntax based functions and globals

* Factored out common stuff from source symbols

* More symbol hammering

* Update ScriptBinding.cs

* Moved stuff

* Update ScriptFunctionSymbol.cs

* More logic

* Update ScriptModuleSymbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Update Binder_Symbol.cs

* Filled out

* Update ScriptBinding.cs

* Update SourceGlobalSymbol.cs

* Bunch of fixes

* Update Binder_Lookup.cs

* ITS ALIVE

* Update ScriptModuleSymbol.cs

* Update ScriptEvalFunctionSymbol.cs

* Shuffled code around

* Fixes

* More fixes

* Fixes

* Rename

* Wrote is complete

* More testing

* Fixed

* Cleanup

* Nicer binder solution

* Simplified
LPeter1997 authored Aug 26, 2024
1 parent 280b45e commit caf883a
Showing 46 changed files with 1,408 additions and 556 deletions.
2 changes: 1 addition & 1 deletion src/Draco.Compiler.Cli/Program.cs
Original file line number Diff line number Diff line change
@@ -136,7 +136,7 @@ private static void RunCommand(FileInfo[] input, DirectoryInfo? rootModule, File
.Select(r => MetadataReference.FromPeStream(r.OpenRead()))
.ToImmutableArray(),
rootModulePath: rootModule?.FullName);
var execResult = ScriptingEngine.Execute(compilation);
var execResult = Script.ExecuteAsProgram(compilation);
if (!EmitDiagnostics(execResult, msbuildDiags))
{
Console.WriteLine($"Result: {execResult.Value}");
2 changes: 1 addition & 1 deletion src/Draco.Compiler.DevHost/Program.cs
Original file line number Diff line number Diff line change
@@ -130,7 +130,7 @@ private static void RunCommand(FileInfo[] input, DirectoryInfo? rootModule, File
.Concat(BclReferences)
.ToImmutableArray(),
rootModulePath: rootModule?.FullName);
var execResult = ScriptingEngine.Execute(compilation);
var execResult = Script.ExecuteAsProgram(compilation);
if (!EmitDiagnostics(execResult))
{
Console.WriteLine($"Result: {execResult.Value}");
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using Draco.Compiler.Api.Scripting;

namespace Draco.Compiler.Tests.Repl;
namespace Draco.Compiler.Tests.Scripting;

public sealed class IsCompleteEntryTests
{
Original file line number Diff line number Diff line change
@@ -2,9 +2,9 @@
using Draco.Compiler.Api.Scripting;
using static Basic.Reference.Assemblies.Net80;

namespace Draco.Compiler.Tests.Repl;
namespace Draco.Compiler.Tests.Scripting;

public sealed class BasicSessionTests
public sealed class ReplSessionTests
{
private static IEnumerable<MetadataReference> BclReferences => ReferenceInfos.All
.Select(r => MetadataReference.FromPeStream(new MemoryStream(r.ImageBytes)));
@@ -15,24 +15,28 @@ public sealed class BasicSessionTests
[InlineData("2 < 3 < 4", true)]
[InlineData("\"asd\" + \"def\"", "asddef")]
[InlineData("\"1 + 2 = \\{1 + 2}\"", "1 + 2 = 3")]
[InlineData("func foo() {}", null)]
[Theory]
public void BasicExpressions(string input, object? output)
{
var replSession = new ReplSession([.. BclReferences]);

var ms = new MemoryStream();
var result = replSession.Evaluate(input);

var writer = new StreamWriter(ms);
writer.WriteLine(input);
writer.Flush();
Assert.True(result.Success);
Assert.Equal(output, result.Value);
}

ms.Position = 0;
var reader = new StreamReader(ms);
[InlineData("func add(x: int32, y: int32) = x + y;")]
[Theory]
public void InvalidEntries(string input)
{
var replSession = new ReplSession([.. BclReferences]);

var result = replSession.Evaluate(reader);
var result = replSession.Evaluate(input);

Assert.True(result.Success);
Assert.Equal(output, result.Value);
Assert.False(result.Success);
Assert.NotEmpty(result.Diagnostics);
}

[Fact]
30 changes: 30 additions & 0 deletions src/Draco.Compiler.Tests/Scripting/ScriptTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
using System.Collections.Immutable;
using Draco.Compiler.Api;
using Draco.Compiler.Api.Scripting;

namespace Draco.Compiler.Tests.Scripting;

public sealed class ScriptTests
{
[Fact]
public void BasicAssignmentAndAddition()
{
// Arrange
var script = Script.Create<int>("""
var x = 3;
var y = 4;
x + y
""",
// TODO: We could factor out BCL refs into some global, we repeat this LINQ a lot in tests
metadataReferences: Basic.Reference.Assemblies.Net80.ReferenceInfos.All
.Select(r => MetadataReference.FromPeStream(new MemoryStream(r.ImageBytes)))
.ToImmutableArray());

// Act
var result = script.Execute();

// Assert
Assert.True(result.Success);
Assert.Equal(7, result.Value);
}
}
15 changes: 14 additions & 1 deletion src/Draco.Compiler/Api/Compilation.cs
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@
using Draco.Compiler.Internal.OptimizingIr;
using Draco.Compiler.Internal.Symbols;
using Draco.Compiler.Internal.Symbols.Metadata;
using Draco.Compiler.Internal.Symbols.Script;
using Draco.Compiler.Internal.Symbols.Source;
using ModuleSymbol = Draco.Compiler.Internal.Symbols.ModuleSymbol;

@@ -389,7 +390,19 @@ private MetadataAssemblySymbol GetMetadataAssembly(MetadataReference metadataRef
this.metadataAssemblies.GetOrAdd(metadataReference, _ => this.BuildMetadataAssembly(metadataReference));

private DeclarationTable BuildDeclarationTable() => new(this);
private ModuleSymbol BuildSourceModule() => new SourceModuleSymbol(this, null, this.DeclarationTable.MergedRoot);

private ModuleSymbol BuildSourceModule()
{
if (this.Flags.HasFlag(CompilationFlags.ScriptingMode))
{
// NOTE: We might want some checks in the constructor?
var syntax = (ScriptEntrySyntax)this.SyntaxTrees.Single().Root;
return new ScriptModuleSymbol(this, null, syntax);
}
// Regular source module
return new SourceModuleSymbol(this, null, this.DeclarationTable.MergedRoot);
}

private ModuleSymbol BuildRootModule() => new MergedModuleSymbol(
containingSymbol: null,
name: string.Empty,
7 changes: 4 additions & 3 deletions src/Draco.Compiler/Api/CompilationFlags.cs
Original file line number Diff line number Diff line change
@@ -14,9 +14,10 @@ public enum CompilationFlags
None = 0,

/// <summary>
/// All defined symbols will be public in the compilation.
/// The compilation is in scripting mode.
///
/// This can be used by things like the REPL to omit visibility.
/// This generally means that it will only consume a single syntax tree with a single
/// script entry syntax.
/// </summary>
ImplicitPublicSymbols = 1 << 0,
ScriptingMode = 1 << 0,
}
10 changes: 10 additions & 0 deletions src/Draco.Compiler/Api/GlobalImports.cs
Original file line number Diff line number Diff line change
@@ -11,6 +11,16 @@ public readonly record struct GlobalImports(
ImmutableArray<string> ModuleImports,
ImmutableArray<(string Name, string FullPath)> ImportAliases)
{
/// <summary>
/// Combines two global import structures into one.
/// </summary>
/// <param name="i1">The first global import structure.</param>
/// <param name="i2">The second global import structure.</param>
/// <returns>The combined global import structure.</returns>
public static GlobalImports Combine(GlobalImports i1, GlobalImports i2) => new(
i1.ModuleImports.AddRange(i2.ModuleImports),
i1.ImportAliases.AddRange(i2.ImportAliases));

/// <summary>
/// True, if this is a default or empty structure.
/// </summary>
173 changes: 32 additions & 141 deletions src/Draco.Compiler/Api/Scripting/ReplSession.cs
Original file line number Diff line number Diff line change
@@ -1,21 +1,15 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using Draco.Compiler.Api.Syntax;
using Draco.Compiler.Internal.Scripting;
using Draco.Compiler.Internal.Syntax;
using static Draco.Compiler.Api.Syntax.SyntaxFactory;
using CompilationUnitSyntax = Draco.Compiler.Api.Syntax.CompilationUnitSyntax;
using DeclarationSyntax = Draco.Compiler.Api.Syntax.DeclarationSyntax;
using ExpressionSyntax = Draco.Compiler.Api.Syntax.ExpressionSyntax;
using ImportDeclarationSyntax = Draco.Compiler.Api.Syntax.ImportDeclarationSyntax;
using ImportPathSyntax = Draco.Compiler.Api.Syntax.ImportPathSyntax;
using MemberImportPathSyntax = Draco.Compiler.Api.Syntax.MemberImportPathSyntax;
using RootImportPathSyntax = Draco.Compiler.Api.Syntax.RootImportPathSyntax;
using ScriptEntrySyntax = Draco.Compiler.Api.Syntax.ScriptEntrySyntax;
using StatementSyntax = Draco.Compiler.Api.Syntax.StatementSyntax;
using SyntaxNode = Draco.Compiler.Api.Syntax.SyntaxNode;

@@ -33,21 +27,11 @@ public sealed class ReplSession
/// <returns>True, if <paramref name="text"/> is a complete entry.</returns>
public static bool IsCompleteEntry(string text)
{
// We add a newline to make sure we don't peek past with trailing trivia if not needed
text = string.Concat(text, Environment.NewLine);
var reader = new DetectOverpeekSourceReader(SourceReader.From(text));
var entry = ParseReplEntry(reader);
// We either haven't overpeeked, or as a special case, we have an empty compilation unit
// which signals an empty entry
return !reader.HasOverpeeked
|| entry.Root is CompilationUnitSyntax { Declarations.Count: 0 };
var tree = SyntaxTree.ParseScript(SourceReader.From(text));
return SyntaxFacts.IsCompleteEntry(tree.Root);
}

private readonly record struct HistoryEntry(Compilation Compilation, Assembly Assembly);

private const string EvalFunctionName = ".eval";

private readonly List<HistoryEntry> previousEntries = [];
private readonly List<Script<object?>> previousEntries = [];
private readonly ReplContext context = new();

public ReplSession(ImmutableArray<MetadataReference> metadataReferences)
@@ -120,7 +104,7 @@ public ExecutionResult<TResult> Evaluate<TResult>(TextReader reader) =>
/// <returns>The execution result.</returns>
internal ExecutionResult<TResult> Evaluate<TResult>(ISourceReader sourceReader)
{
var tree = ParseReplEntry(sourceReader);
var tree = SyntaxTree.ParseScript(sourceReader);

// Check for syntax errors
if (tree.HasErrors)
@@ -140,138 +124,45 @@ internal ExecutionResult<TResult> Evaluate<TResult>(ISourceReader sourceReader)
/// <returns>The execution result.</returns>
public ExecutionResult<TResult> Evaluate<TResult>(SyntaxNode node)
{
// Check for an empty entry
if (node is CompilationUnitSyntax { Declarations.Count: 0 })
{
return ExecutionResult.Success(default(TResult)!);
}

// Check for imports
if (node is ImportDeclarationSyntax import)
{
this.context.AddImport(ExtractImportPath(import.Path));
return ExecutionResult.Success(default(TResult)!);
}

// Translate to a runnable function
var decl = node switch
{
ExpressionSyntax expr => this.ToDeclaration(expr),
StatementSyntax stmt => this.ToDeclaration(stmt),
DeclarationSyntax d => d,
_ => throw new ArgumentOutOfRangeException(nameof(node)),
};

// Wrap in a tree
var tree = this.ToSyntaxTree(decl);

// Find the relocated node in the tree, we need this to shift diagnostics
var relocatedNode = node switch
{
ExpressionSyntax expr => tree.FindInChildren<ExpressionSyntax>() as SyntaxNode,
StatementSyntax stmt => tree.FindInChildren<StatementSyntax>(),
DeclarationSyntax d => tree.FindInChildren<DeclarationSyntax>(1),
_ => throw new ArgumentOutOfRangeException(nameof(node)),
};
var tree = ToSyntaxTree(node);

// Make compilation
var compilation = this.MakeCompilation(tree);
// Create a script
var script = this.MakeScript(tree);

// Emit the assembly
var peStream = new MemoryStream();
var result = compilation.Emit(peStream: peStream);
// Try to execute
var result = script.Execute();

// Transform all the diagnostics
var diagnostics = result.Diagnostics
.Select(d => d.RelativeTo(relocatedNode))
.ToImmutableArray();

// Check for errors
if (!result.Success) return ExecutionResult.Fail<TResult>(diagnostics);

// If it was a non-empty declaration, track it
if (node is DeclarationSyntax)
{
var semanticModel = compilation.GetSemanticModel(tree);
var symbol = semanticModel.GetDeclaredSymbolInternal(relocatedNode);
if (symbol is not null) this.context.AddSymbol(symbol);
}
// If failed, bail out
if (!result.Success) return ExecutionResult.Fail<TResult>(result.Diagnostics);

// We need to load the assembly in the current context
peStream.Position = 0;
var assembly = this.context.LoadAssembly(peStream);
// Stash the entry
this.previousEntries.Add(script);

// Stash it for future use
this.previousEntries.Add(new HistoryEntry(Compilation: compilation, Assembly: assembly));
// We want to stash the exports of the script
this.context.AddAll(script.GlobalImports);
// And the metadata references
this.context.AddMetadataReference(MetadataReference.FromAssembly(script.Assembly!));

// Register the metadata reference
this.context.AddMetadataReference(MetadataReference.FromAssembly(assembly));

// Retrieve the main module
var mainModule = assembly.GetType(compilation.RootModulePath);
Debug.Assert(mainModule is not null);

// Run the eval function
var eval = mainModule.GetMethod(EvalFunctionName);
if (eval is not null)
{
var value = (TResult?)eval.Invoke(null, null);
return ExecutionResult.Success(value!, diagnostics);
}

// This happens with declarations, nothing to run
return ExecutionResult.Success(default(TResult)!, diagnostics);
// Return result
return ExecutionResult.Success((TResult)result.Value!);
}

// func .eval(): object = decl;
private DeclarationSyntax ToDeclaration(ExpressionSyntax expr) => FunctionDeclaration(
EvalFunctionName,
ParameterList(),
NameType("object"),
InlineFunctionBody(expr));

// func .eval() = stmt;
private DeclarationSyntax ToDeclaration(StatementSyntax stmt) => FunctionDeclaration(
EvalFunctionName,
ParameterList(),
null,
InlineFunctionBody(StatementExpression(stmt)));

private SyntaxTree ToSyntaxTree(DeclarationSyntax decl) => SyntaxTree.Create(CompilationUnit(decl));

private Compilation MakeCompilation(SyntaxTree tree) => Compilation.Create(
syntaxTrees: [tree],
metadataReferences: this.context.MetadataReferences,
flags: CompilationFlags.ImplicitPublicSymbols,
private Script<object?> MakeScript(SyntaxTree tree) => Script.Create(
syntaxTree: tree,
globalImports: this.context.GlobalImports,
rootModulePath: $"Context{this.previousEntries.Count}",
assemblyName: $"ReplAssembly{this.previousEntries.Count}",
metadataAssemblies: this.previousEntries.Count == 0
metadataReferences: this.context.MetadataReferences,
previousCompilation: this.previousEntries.Count == 0
? null
: this.previousEntries[^1].Compilation.MetadataAssembliesDict);

private static SyntaxTree ParseReplEntry(ISourceReader sourceReader)
{
var syntaxDiagnostics = new SyntaxDiagnosticTable();

// Construct a lexer
var lexer = new Lexer(sourceReader, syntaxDiagnostics);
// Construct a token source
var tokenSource = TokenSource.From(lexer);
// Construct a parser
var parser = new Parser(tokenSource, syntaxDiagnostics, parserMode: ParserMode.Repl);
// Parse a repl entry
var node = parser.ParseReplEntry();
// Make it into a tree
var tree = SyntaxTree.Create(node);

return tree;
}
: this.previousEntries[^1].Compilation,
assemblyLoadContext: this.context.AssemblyLoadContext);

private static string ExtractImportPath(ImportPathSyntax path) => path switch
private static SyntaxTree ToSyntaxTree(SyntaxNode node) => node switch
{
RootImportPathSyntax root => root.Name.Text,
MemberImportPathSyntax member => $"{ExtractImportPath(member.Accessed)}.{member.Member.Text}",
_ => throw new ArgumentOutOfRangeException(nameof(path)),
ScriptEntrySyntax se => SyntaxTree.Create(se),
ExpressionSyntax e => SyntaxTree.Create(ScriptEntry(SyntaxList<StatementSyntax>(), e, EndOfInput)),
StatementSyntax s => SyntaxTree.Create(ScriptEntry(SyntaxList(s), null, EndOfInput)),
DeclarationSyntax d => SyntaxTree.Create(ScriptEntry(SyntaxList<StatementSyntax>(DeclarationStatement(d)), null, EndOfInput)),
_ => throw new ArgumentOutOfRangeException(nameof(node)),
};
}
Loading

0 comments on commit caf883a

Please sign in to comment.