Basic benchmarking (#319)

* Rename, created project

* Basic framework works

* Create 99bottles.draco

* Update LexerBenchmarks.cs

* Added parser tests

* Basic e2e benchmarking

* Namespace sync

* Cleanup

* PR comment

* Update SyntaxBenchmarks.cs

src/Draco.Compiler.Benchmarks/Draco.Compiler.Benchmarks.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <!-- Copy benchmark inputs -->
+  <ItemGroup>
+    <Content Include="benchmarks\**">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Draco.Compiler\Draco.Compiler.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.8" />
+    <PackageReference Include="Basic.Reference.Assemblies.Net70" Version="1.4.1" />
+  </ItemGroup>
+
+</Project>

src/Draco.Compiler.Benchmarks/E2eBenchmarks.cs

@@ -0,0 +1,36 @@
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+using Draco.Compiler.Api;
+using Draco.Compiler.Api.Syntax;
+
+namespace Draco.Compiler.Benchmarks;
+
+public class E2eBenchmarks : FolderBenchmarkBase
+{
+    private MemoryStream peStream = null!;
+
+    public E2eBenchmarks()
+        : base("e2e")
+    {
+    }
+
+    [IterationSetup]
+    public void Setup()
+    {
+        this.peStream = new();
+    }
+
+    [Benchmark]
+    public EmitResult Compile()
+    {
+        var syntaxTree = SyntaxTree.Parse(this.Input.Code, Path.GetFullPath(this.Input.Path));
+        var compilation = Compilation.Create(
+            syntaxTrees: ImmutableArray.Create(syntaxTree),
+            metadataReferences: Basic.Reference.Assemblies.Net70.ReferenceInfos.All
+                .Select(r => MetadataReference.FromPeStream(new MemoryStream(r.ImageBytes)))
+                .ToImmutableArray());
+        return compilation.Emit(peStream: this.peStream);
+    }
+}

src/Draco.Compiler.Benchmarks/FolderBenchmarkBase.cs

@@ -0,0 +1,23 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using BenchmarkDotNet.Attributes;
+
+namespace Draco.Compiler.Benchmarks;
+
+public abstract class FolderBenchmarkBase
+{
+    [ParamsSource(nameof(GetSourcesFromFolder))]
+    public SourceCodeParameter Input { get; set; } = null!;
+
+    private readonly string path;
+
+    protected FolderBenchmarkBase(string path)
+    {
+        this.path = Path.Join("benchmarks", path);
+    }
+
+    public IEnumerable<SourceCodeParameter> GetSourcesFromFolder() => Directory
+        .GetFiles(this.path)
+        .Select(SourceCodeParameter.FromPath);
+}

src/Draco.Compiler.Benchmarks/Program.cs

@@ -0,0 +1,9 @@
+using BenchmarkDotNet.Running;
+
+namespace Draco.Compiler.Benchmarks;
+
+internal class Program
+{
+    internal static void Main(string[] args) =>
+        BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+}

src/Draco.Compiler.Benchmarks/SourceCodeParameter.cs

@@ -0,0 +1,10 @@
+using System.IO;
+
+namespace Draco.Compiler.Benchmarks;
+
+public sealed record class SourceCodeParameter(string Path, string Code)
+{
+    public static SourceCodeParameter FromPath(string path) => new(path, File.ReadAllText(path));
+
+    public override string ToString() => System.IO.Path.GetFileName(this.Path);
+}

src/Draco.Compiler.Benchmarks/SyntaxBenchmarks.cs

@@ -0,0 +1,96 @@
+using System;
+using System.Collections.Generic;
+using BenchmarkDotNet.Attributes;
+using Draco.Compiler.Api.Syntax;
+using Draco.Compiler.Internal.Syntax;
+using CompilationUnitSyntax = Draco.Compiler.Internal.Syntax.CompilationUnitSyntax;
+using SyntaxToken = Draco.Compiler.Internal.Syntax.SyntaxToken;
+
+namespace Draco.Compiler.Benchmarks;
+
+public class SyntaxBenchmarks : FolderBenchmarkBase
+{
+    private SyntaxToken[] tokens = null!;
+
+    private Lexer lexer = null!;
+    private Parser parser = null!;
+
+    public SyntaxBenchmarks()
+        : base("syntax")
+    {
+    }
+
+    [GlobalSetup]
+    public void GlobalSetup()
+    {
+        this.tokens = LexFromBenchmarkParameter(this.Input);
+    }
+
+    [IterationSetup(Target = nameof(Lex))]
+    public void LexSetup()
+    {
+        var sourceReader = SourceReader.From(this.Input.Code);
+        var syntaxDiagnostics = new SyntaxDiagnosticTable();
+        this.lexer = new Lexer(sourceReader, syntaxDiagnostics);
+    }
+
+    [IterationSetup(Target = nameof(Parse))]
+    public void ParseSetup()
+    {
+        var tokenSource = TokenSource.From(this.tokens.AsMemory());
+        var syntaxDiagnostics = new SyntaxDiagnosticTable();
+        this.parser = new Parser(tokenSource, syntaxDiagnostics);
+    }
+
+    [IterationSetup(Target = nameof(ParseWithStreamingLexer))]
+    public void ParseWithStreamingLexerSetup()
+    {
+        var syntaxDiagnostics = new SyntaxDiagnosticTable();
+
+        var sourceReader = SourceReader.From(this.Input.Code);
+        this.lexer = new Lexer(sourceReader, syntaxDiagnostics);
+
+        var tokenSource = TokenSource.From(this.lexer);
+        this.parser = new Parser(tokenSource, syntaxDiagnostics);
+    }
+
+    [Benchmark]
+    public int Lex()
+    {
+        var count = 0;
+        while (true)
+        {
+            var token = this.lexer.Lex();
+            ++count;
+            if (token.Kind == TokenKind.EndOfInput) break;
+        }
+        return count;
+    }
+
+    [Benchmark]
+    public object Parse() =>
+        this.parser.ParseCompilationUnit();
+
+    [Benchmark]
+    public object ParseWithStreamingLexer() =>
+        this.parser.ParseCompilationUnit();
+
+    private static SyntaxToken[] LexFromBenchmarkParameter(SourceCodeParameter parameter)
+    {
+        var sourceReader = SourceReader.From(parameter.Code);
+        var syntaxDiagnostics = new SyntaxDiagnosticTable();
+        var lexer = new Lexer(sourceReader, syntaxDiagnostics);
+
+        var result = new List<SyntaxToken>();
+
+        while (true)
+        {
+            var token = lexer.Lex();
+            result.Add(token);
+
+            if (token.Kind == TokenKind.EndOfInput) break;
+        }
+
+        return result.ToArray();
+    }
+}

src/Draco.Compiler.Benchmarks/benchmarks/e2e/99bottles.draco

@@ -0,0 +1,29 @@
+import System.Console;
+
+func bottles(n: int32): string =
+    if (n == 0) "no more bottles"
+    else if (n == 1) "1 bottle"
+    else "\{n} bottles";
+
+func capitalize(s: string): string =
+    "\{s.Substring(0, 1).ToUpper()}\{s.Substring(1)}";
+
+func main() {
+    var i = 99;
+    while (i > 0) {
+        WriteLine("""
+            \{capitalize(bottles(i))} of beer on the wall,
+            \{bottles(i)} of beer.
+            Take one down, pass it around,
+            \{bottles(i - 1)} of beer on the wall.
+
+            """);
+        i -= 1;
+    }
+    WriteLine("""
+        No more bottles of beer on the wall,
+        no more bottles of beer.
+        Go to the store, buy some more,
+        99 bottles of beer on the wall.
+        """);
+}

src/Draco.Compiler.Benchmarks/benchmarks/e2e/helloworld.draco

@@ -0,0 +1,5 @@
+import System.Console;
+
+public func main() {
+    WriteLine("Hello, World!");
+}

src/Draco.Compiler.Benchmarks/benchmarks/syntax/99bottles.draco

@@ -0,0 +1,29 @@
+import System.Console;
+
+func bottles(n: int32): string =
+    if (n == 0) "no more bottles"
+    else if (n == 1) "1 bottle"
+    else "\{n} bottles";
+
+func capitalize(s: string): string =
+    "\{s.Substring(0, 1).ToUpper()}\{s.Substring(1)}";
+
+func main() {
+    var i = 99;
+    while (i > 0) {
+        WriteLine("""
+            \{capitalize(bottles(i))} of beer on the wall,
+            \{bottles(i)} of beer.
+            Take one down, pass it around,
+            \{bottles(i - 1)} of beer on the wall.
+
+            """);
+        i -= 1;
+    }
+    WriteLine("""
+        No more bottles of beer on the wall,
+        no more bottles of beer.
+        Go to the store, buy some more,
+        99 bottles of beer on the wall.
+        """);
+}

src/Draco.Compiler.Benchmarks/benchmarks/syntax/helloworld.draco

@@ -0,0 +1,5 @@
+import System.Console;
+
+public func main() {
+    WriteLine("Hello, World!");
+}

src/Draco.Fuzzer/Components/ComponentFuzzerBase.cs → src/Draco.Compiler.Fuzzer/Components/ComponentFuzzerBase.cs

@@ -1,6 +1,6 @@
-using Draco.Fuzzer.Generators;
+using Draco.Compiler.Fuzzer.Generators;
 
-namespace Draco.Fuzzer.Components;
+namespace Draco.Compiler.Fuzzer.Components;
 
 /// <summary>
 /// Utility base-class for component fuzzers.

src/Draco.Fuzzer/Components/E2eFuzzer.cs → src/Draco.Compiler.Fuzzer/Components/E2eFuzzer.cs

@@ -2,9 +2,9 @@
 using Draco.Compiler.Api;
 using Draco.Compiler.Api.Scripting;
 using Draco.Compiler.Api.Syntax;
-using Draco.Fuzzer.Generators;
+using Draco.Compiler.Fuzzer.Generators;
 
-namespace Draco.Fuzzer.Components;
+namespace Draco.Compiler.Fuzzer.Components;
 
 /// <summary>
 /// Fuzzes the compiler end-to-end.

src/Draco.Fuzzer/Components/IComponentFuzzer.cs → src/Draco.Compiler.Fuzzer/Components/IComponentFuzzer.cs

@@ -1,4 +1,4 @@
-namespace Draco.Fuzzer.Components;
+namespace Draco.Compiler.Fuzzer.Components;
 
 /// <summary>
 /// Represents the fuzzer of a compiler component.

src/Draco.Fuzzer/Components/LexerFuzzer.cs → src/Draco.Compiler.Fuzzer/Components/LexerFuzzer.cs

@@ -1,8 +1,8 @@
 using Draco.Compiler.Api.Syntax;
+using Draco.Compiler.Fuzzer.Generators;
 using Draco.Compiler.Internal.Syntax;
-using Draco.Fuzzer.Generators;
 
-namespace Draco.Fuzzer.Components;
+namespace Draco.Compiler.Fuzzer.Components;
 
 /// <summary>
 /// Fuzzes the lexer.

src/Draco.Fuzzer/Components/ParserFuzzer.cs → src/Draco.Compiler.Fuzzer/Components/ParserFuzzer.cs

@@ -1,8 +1,8 @@
 using System.Collections.Immutable;
+using Draco.Compiler.Fuzzer.Generators;
 using Draco.Compiler.Internal.Syntax;
-using Draco.Fuzzer.Generators;
 
-namespace Draco.Fuzzer.Components;
+namespace Draco.Compiler.Fuzzer.Components;
 
 /// <summary>
 /// Fuzzes the parser.

src/Draco.Fuzzer/CrashException.cs → src/Draco.Compiler.Fuzzer/CrashException.cs

@@ -1,4 +1,4 @@
-namespace Draco.Fuzzer;
+namespace Draco.Compiler.Fuzzer;
 
 /// <summary>
 /// Represents an exception coming from a non-incremental change.

src/Draco.Fuzzer/Generators/Charsets.cs → src/Draco.Compiler.Fuzzer/Generators/Charsets.cs

@@ -1,4 +1,4 @@
-namespace Draco.Fuzzer.Generators;
+namespace Draco.Compiler.Fuzzer.Generators;
 
 /// <summary>
 /// Character set constants.

src/Draco.Fuzzer/Generators/Generator.cs → src/Draco.Compiler.Fuzzer/Generators/Generator.cs

@@ -1,7 +1,7 @@
 using System.Collections.Immutable;
 using System.Globalization;
 
-namespace Draco.Fuzzer.Generators;
+namespace Draco.Compiler.Fuzzer.Generators;
 
 /// <summary>
 /// Utility functions for generation.

src/Draco.Fuzzer/Generators/IGenerator.cs → src/Draco.Compiler.Fuzzer/Generators/IGenerator.cs

@@ -1,4 +1,4 @@
-namespace Draco.Fuzzer.Generators;
+namespace Draco.Compiler.Fuzzer.Generators;
 
 /// <summary>
 /// Generates input for the fuzzer components.

src/Draco.Fuzzer/Generators/SequenceGenerationSettings.cs → src/Draco.Compiler.Fuzzer/Generators/SequenceGenerationSettings.cs

@@ -1,4 +1,4 @@
-namespace Draco.Fuzzer.Generators;
+namespace Draco.Compiler.Fuzzer.Generators;
 
 /// <summary>
 /// Settings for sequence generation.

src/Draco.Fuzzer/Generators/SequenceGenerator.cs → src/Draco.Compiler.Fuzzer/Generators/SequenceGenerator.cs

@@ -1,6 +1,6 @@
 using System.Collections.Immutable;
 
-namespace Draco.Fuzzer.Generators;
+namespace Draco.Compiler.Fuzzer.Generators;
 
 /// <summary>
 /// An <see cref="IGenerator{T}"/> that generates a sequence based on another generator.

src/Draco.Fuzzer/Generators/TokenGenerator.cs → src/Draco.Compiler.Fuzzer/Generators/TokenGenerator.cs

@@ -3,7 +3,7 @@
 using SyntaxToken = Draco.Compiler.Internal.Syntax.SyntaxToken;
 using SyntaxTrivia = Draco.Compiler.Internal.Syntax.SyntaxTrivia;
 
-namespace Draco.Fuzzer.Generators;
+namespace Draco.Compiler.Fuzzer.Generators;
 
 /// <summary>
 /// Generates a random valid token.

src/Draco.Fuzzer/Generators/TriviaGenerator.cs → src/Draco.Compiler.Fuzzer/Generators/TriviaGenerator.cs

@@ -1,7 +1,7 @@
 using Draco.Compiler.Api.Syntax;
 using SyntaxTrivia = Draco.Compiler.Internal.Syntax.SyntaxTrivia;
 
-namespace Draco.Fuzzer.Generators;
+namespace Draco.Compiler.Fuzzer.Generators;
 
 /// <summary>
 /// Generates a random valid syntax trivia.

src/Draco.Fuzzer/MutationException.cs → src/Draco.Compiler.Fuzzer/MutationException.cs

@@ -1,4 +1,4 @@
-namespace Draco.Fuzzer;
+namespace Draco.Compiler.Fuzzer;
 
 /// <summary>
 /// Represents an exception coming from an incremental change.