From 1cfc12ebdb7da2bbccfb4c482c4092a2f598214b Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Sat, 14 Oct 2023 17:55:13 +0200
Subject: [PATCH 01/29] wip: start work on top-down name resolution

---
 src/analysis.zig | 306 +++++++++++++++++++++++++++++++++--------------
 src/main.zig     |  12 +-
 src/syntax.zig   |   5 +
 3 files changed, 231 insertions(+), 92 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index f4f8466..a85649a 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -148,7 +148,10 @@ test "typeOf function" {
 }
 
 fn expectTypeFormat(source: []const u8, types: []const []const u8) !void {
-    const allocator = std.testing.allocator;
+    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+    defer arena.deinit();
+
+    const allocator = arena.allocator();
 
     var workspace = try Workspace.init(allocator);
     defer workspace.deinit();
@@ -170,7 +173,7 @@ fn expectTypeFormat(source: []const u8, types: []const []const u8) !void {
         var references = std.ArrayList(Reference).init(allocator);
         defer references.deinit();
 
-        try findDefinition(document, cursor.definition, &references);
+        try findDefinition(allocator, document, cursor.definition, &references);
         if (references.items.len != 1) return error.InvalidReference;
         const ref = references.items[0];
 
@@ -184,19 +187,21 @@ fn expectTypeFormat(source: []const u8, types: []const []const u8) !void {
 }
 
 // Given a node in the given parse tree, attempts to find the node(s) it references.
-pub fn findDefinition(document: *Document, node: u32, references: *std.ArrayList(Reference)) !void {
-    const workspace = document.workspace;
-    const allocator = workspace.allocator;
-
+pub fn findDefinition(
+    arena: std.mem.Allocator,
+    document: *Document,
+    node: u32,
+    references: *std.ArrayList(Reference),
+) !void {
     const parse_tree = try document.parseTree();
     const tree = parse_tree.tree;
 
     const name = nodeName(tree, node, document.source()) orelse return;
 
-    var symbols = std.ArrayList(Reference).init(allocator);
+    var symbols = std.ArrayList(Reference).init(arena);
     defer symbols.deinit();
 
-    try visibleSymbols(document, node, &symbols);
+    try visibleSymbols(arena, document, node, &symbols);
 
     for (symbols.items) |symbol| {
         const parsed = try symbol.document.parseTree();
@@ -215,7 +220,12 @@ fn inFileRoot(tree: Tree, node: u32) bool {
     return tree.tag(parent) == .file;
 }
 
-pub fn visibleFields(document: *Document, node: u32, symbols: *std.ArrayList(Reference)) !void {
+pub fn visibleFields(
+    arena: std.mem.Allocator,
+    document: *Document,
+    node: u32,
+    symbols: *std.ArrayList(Reference),
+) !void {
     const name = blk: {
         const parsed = try document.parseTree();
         const tag = parsed.tree.tag(node);
@@ -237,9 +247,8 @@ pub fn visibleFields(document: *Document, node: u32, symbols: *std.ArrayList(Ref
         break :blk first;
     };
 
-    var name_definitions = std.ArrayList(Reference).init(document.workspace.allocator);
-    defer name_definitions.deinit();
-    try findDefinition(document, name, &name_definitions);
+    var name_definitions = std.ArrayList(Reference).init(arena);
+    try findDefinition(arena, document, name, &name_definitions);
     if (name_definitions.items.len == 0) return;
 
     var references = std.ArrayList(Reference).init(document.workspace.allocator);
@@ -272,7 +281,7 @@ pub fn visibleFields(document: *Document, node: u32, symbols: *std.ArrayList(Ref
                     },
                     else => {
                         const identifier = specifier.underlyingName(tree) orelse continue;
-                        try findDefinition(reference.document, identifier.node, &references);
+                        try findDefinition(arena, reference.document, identifier.node, &references);
                     },
                 }
                 continue;
@@ -296,85 +305,202 @@ pub fn visibleFields(document: *Document, node: u32, symbols: *std.ArrayList(Ref
     }
 }
 
-/// Get a list of all symbols visible starting from the given syntax node
+pub const Scope = struct {
+    const ScopeId = u32;
+
+    allocator: std.mem.Allocator,
+    symbols: std.StringArrayHashMapUnmanaged(Symbol) = .{},
+    active_scopes: std.ArrayListUnmanaged(ScopeId) = .{},
+    next_scope: ScopeId = 0,
+
+    const Symbol = struct {
+        /// In which scope this item is valid.
+        scope: ScopeId,
+        /// The syntax node this name refers to.
+        reference: Reference,
+        /// Pointer to any shadowed symbols.
+        shadowed: ?*Symbol = null,
+    };
+
+    pub fn begin(self: *@This()) !void {
+        try self.active_scopes.append(self.allocator, self.next_scope);
+        self.next_scope += 1;
+    }
+
+    pub fn end(self: *@This()) void {
+        _ = self.active_scopes.pop();
+    }
+
+    fn currentScope(self: *const @This()) ScopeId {
+        const active = self.active_scopes.items;
+        return active[active.len - 1];
+    }
+
+    pub fn add(self: *@This(), name: []const u8, reference: Reference) !void {
+        const result = try self.symbols.getOrPut(self.allocator, name);
+        errdefer self.symbols.swapRemoveAt(result.index);
+
+        var shadowed: ?*Symbol = null;
+        if (result.found_existing) {
+            const copy = try self.allocator.create(Symbol);
+            copy.* = result.value_ptr.*;
+            shadowed = copy;
+        }
+
+        result.value_ptr.* = .{
+            .scope = self.currentScope(),
+            .reference = reference,
+            .shadowed = shadowed,
+        };
+    }
+
+    pub fn isActive(self: *const @This(), scope: ScopeId) bool {
+        for (self.active_scopes.items) |active| {
+            if (scope == active) return true;
+        }
+        return false;
+    }
+
+    pub fn getVisible(self: *const @This(), symbols: *std.ArrayList(Reference)) !void {
+        try symbols.ensureUnusedCapacity(self.symbols.count());
+        for (self.symbols.values()) |*value| {
+            var current: ?*Symbol = value;
+            while (current) |symbol| : (current = symbol.shadowed) {
+                if (!self.isActive(symbol.scope)) continue;
+                try symbols.append(symbol.reference);
+            }
+        }
+    }
+};
+
+/// Get a list of all symbols visible starting from the given syntax node.
 pub fn visibleSymbols(
+    arena: std.mem.Allocator,
     start_document: *Document,
     start_node: u32,
     symbols: *std.ArrayList(Reference),
 ) !void {
-    const workspace = start_document.workspace;
-    var arena = std.heap.ArenaAllocator.init(workspace.allocator);
-    defer arena.deinit();
-    const allocator = arena.allocator();
+    _ = start_node;
+    var scope = Scope{ .allocator = arena };
+    try scope.begin();
+    defer scope.end();
 
-    var visited_documents = std.AutoHashMap(*Document, void).init(allocator);
-    defer visited_documents.deinit();
-
-    var stack = std.ArrayList(struct { document: *Document, node: ?u32 }).init(allocator);
-    defer stack.deinit();
-
-    try stack.append(.{ .document = start_document, .node = start_node });
-    try visited_documents.put(start_document, {});
-
-    while (stack.popOrNull()) |current| {
-        const document = current.document;
-        const parse_tree = try document.parseTree();
-        const tree = parse_tree.tree;
-        const node = current.node orelse tree.rootIndex();
-
-        const symbols_start = symbols.items.len;
-        try visibleSymbolsTree(document, tree.parent(node) orelse node, symbols);
-        const unique = try partitonUniqueSymbols(allocator, symbols.items[symbols_start..], tree.nodes.len);
-        symbols.items.len = symbols_start + unique;
-
-        if (std.fs.path.dirname(document.path)) |document_dir| {
-            const node_end = if (tree.tag(node) == .file)
-                std.math.maxInt(u32)
-            else
-                tree.nodeSpanExtreme(node, .start);
-
-            // parse include directives
-            for (parse_tree.ignored, parse_tree.tree.ignoredStart()..) |ignored, ignored_index| {
-                // only process directives before the node:
-                if (ignored.end > node_end) break;
-
-                const line = document.source()[ignored.start..ignored.end];
-                const directive = parse.parsePreprocessorDirective(line) orelse continue;
-                switch (directive) {
-                    .include => |include| {
-                        const relative_path = line[include.path.start..include.path.end];
-
-                        const uri = if (std.fs.path.isAbsolute(relative_path))
-                            try util.uriFromPath(allocator, relative_path)
-                        else blk: {
-                            const absolute = try std.fs.path.join(allocator, &.{ document_dir, relative_path });
-                            defer allocator.free(absolute);
-                            break :blk try util.uriFromPath(allocator, absolute);
-                        };
-                        defer allocator.free(uri);
-
-                        const included_document = workspace.getOrLoadDocument(.{ .uri = uri }) catch |err| {
-                            std.log.err("could not open '{'}': {s}", .{
-                                std.zig.fmtEscapes(uri),
-                                @errorName(err),
-                            });
-                            continue;
-                        };
-
-                        const entry = try visited_documents.getOrPut(included_document);
-                        if (entry.found_existing) continue;
-
-                        try stack.append(.{ .document = included_document, .node = null });
-                    },
-                    .define => {
-                        try symbols.append(Reference{
-                            .document = document,
-                            .node = @intCast(ignored_index),
-                            .parent_declaration = @intCast(ignored_index),
-                        });
-                    },
-                }
+    // collect global symbols:
+    {
+        var documents = try std.ArrayList(*Document).initCapacity(arena, 8);
+        defer documents.deinit();
+
+        try documents.append(start_document);
+        try findIncludedDocumentsRecursive(arena, &documents);
+
+        for (documents.items) |document| {
+            try collectGlobalSymbols(&scope, document);
+        }
+    }
+
+    try scope.getVisible(symbols);
+}
+
+pub fn collectGlobalSymbols(scope: *Scope, document: *Document) !void {
+    const parsed = try document.parseTree();
+    const tree = parsed.tree;
+
+    const children = tree.children(tree.root);
+    for (children.start..children.end) |child| {
+        const global = syntax.AnyDeclaration.tryExtract(tree, @intCast(child)) orelse continue;
+        try collectSymbols(scope, document, tree, global);
+    }
+}
+
+fn collectSymbols(
+    scope: *Scope,
+    document: *Document,
+    tree: parse.Tree,
+    any: syntax.AnyDeclaration,
+) !void {
+    switch (any) {
+        .function => |function| {
+            const ident = function.get(.identifier, tree) orelse return;
+            try scope.add(ident.text(document.source(), tree), .{
+                .document = document,
+                .node = ident.node,
+                .parent_declaration = function.node,
+            });
+        },
+        .variable => |declaration| {
+            const variables = declaration.get(.variables, tree) orelse return;
+            var iterator = variables.iterator();
+            while (iterator.next(tree)) |variable| {
+                const name = variable.get(.name, tree) orelse return;
+                const ident = name.getIdentifier(tree) orelse return;
+                try scope.add(ident.text(document.source(), tree), .{
+                    .document = document,
+                    .node = ident.node,
+                    .parent_declaration = declaration.node,
+                });
             }
+        },
+        else => {
+            std.log.warn("TODO: collect symbols from {s}", .{@tagName(any)});
+        },
+    }
+}
+
+/// Appends the set of documents which are visible (recursively) from any of the documents in the list.
+fn findIncludedDocumentsRecursive(
+    arena: std.mem.Allocator,
+    documents: *std.ArrayList(*Document),
+) !void {
+    var i: usize = 0;
+    while (i < documents.items.len) : (i += 1) {
+        try findIncludedDocuments(arena, documents.items[i], documents);
+    }
+}
+
+/// Appends the set of documents which are visible (directly) from the given document.
+fn findIncludedDocuments(
+    arena: std.mem.Allocator,
+    start: *Document,
+    documents: *std.ArrayList(*Document),
+) !void {
+    const parsed = try start.parseTree();
+
+    const document_dir = std.fs.path.dirname(start.path) orelse return;
+
+    for (parsed.tree.ignored()) |extra| {
+        const line = start.source()[extra.start..extra.end];
+        const directive = parse.parsePreprocessorDirective(line) orelse continue;
+        switch (directive) {
+            .include => |include| {
+                var include_path = line[include.path.start..include.path.end];
+
+                const is_relative = !std.fs.path.isAbsolute(include_path);
+
+                var absolute_path = include_path;
+                if (is_relative) absolute_path = try std.fs.path.join(arena, &.{ document_dir, include_path });
+                defer if (is_relative) arena.free(absolute_path);
+
+                const uri = try util.uriFromPath(arena, absolute_path);
+                defer arena.free(uri);
+
+                const included_document = start.workspace.getOrLoadDocument(.{ .uri = uri }) catch |err| {
+                    std.log.err("could not open '{'}': {s}", .{
+                        std.zig.fmtEscapes(uri),
+                        @errorName(err),
+                    });
+                    continue;
+                };
+
+                for (documents.items) |document| {
+                    if (document == included_document) {
+                        // document has already been included.
+                        break;
+                    }
+                } else {
+                    try documents.append(included_document);
+                }
+            },
+            else => continue,
         }
     }
 }
@@ -626,6 +752,9 @@ fn expectDefinitionIsFound(source: []const u8) !void {
     var workspace = try Workspace.init(std.testing.allocator);
     defer workspace.deinit();
 
+    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+    defer arena.deinit();
+
     const document = try workspace.getOrCreateDocument(.{ .uri = "file://test.glsl", .version = 0 });
     try document.replaceAll(source);
 
@@ -636,7 +765,7 @@ fn expectDefinitionIsFound(source: []const u8) !void {
         for (cursor.usages.slice()) |usage| {
             var references = std.ArrayList(Reference).init(workspace.allocator);
             defer references.deinit();
-            try findDefinition(document, usage, &references);
+            try findDefinition(arena.allocator(), document, usage, &references);
             if (references.items.len == 0) return error.ReferenceNotFound;
             if (references.items.len > 1) return error.MultipleDefinitions;
             const ref = references.items[0];
@@ -650,6 +779,9 @@ fn expectDefinitionIsNotFound(source: []const u8) !void {
     var workspace = try Workspace.init(std.testing.allocator);
     defer workspace.deinit();
 
+    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+    defer arena.deinit();
+
     const document = try workspace.getOrCreateDocument(.{ .uri = "file://test.glsl", .version = 0 });
     try document.replaceAll(source);
 
@@ -660,7 +792,7 @@ fn expectDefinitionIsNotFound(source: []const u8) !void {
         for (cursor.usages.slice()) |usage| {
             var references = std.ArrayList(Reference).init(workspace.allocator);
             defer references.deinit();
-            try findDefinition(document, usage, &references);
+            try findDefinition(arena.allocator(), document, usage, &references);
             if (references.items.len != 0) {
                 const ref = references.items[0];
                 std.debug.print("found unexpected reference: {s}:{}\n", .{ ref.document.path, ref.node });
diff --git a/src/main.zig b/src/main.zig
index 0cd8fa8..b65e73c 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -269,6 +269,7 @@ pub const Channel = union(enum) {
 
 const State = struct {
     allocator: std.mem.Allocator,
+
     channel: *std.io.BufferedWriter(4096, Channel.Writer),
     running: bool = true,
     initialized: bool = false,
@@ -567,13 +568,12 @@ pub const Dispatch = struct {
         var has_fields = false;
 
         if (try document.nodeBeforeCursor(position)) |node| {
-            var symbols = std.ArrayList(analysis.Reference).init(state.allocator);
-            defer symbols.deinit();
+            var symbols = std.ArrayList(analysis.Reference).init(arena);
 
-            try analysis.visibleFields(document, node, &symbols);
+            try analysis.visibleFields(arena, document, node, &symbols);
             has_fields = symbols.items.len != 0;
 
-            if (!has_fields) try analysis.visibleSymbols(document, node, &symbols);
+            if (!has_fields) try analysis.visibleSymbols(arena, document, node, &symbols);
 
             try completions.ensureUnusedCapacity(symbols.items.len);
 
@@ -740,7 +740,9 @@ pub const Dispatch = struct {
         var references = std.ArrayList(analysis.Reference).init(state.allocator);
         defer references.deinit();
 
-        try analysis.findDefinition(document, source_node, &references);
+        var arena = std.heap.ArenaAllocator.init(state.allocator);
+        defer arena.deinit();
+        try analysis.findDefinition(arena.allocator(), document, source_node, &references);
 
         if (references.items.len == 0) {
             std.log.debug("could not find definition", .{});
diff --git a/src/syntax.zig b/src/syntax.zig
index e1b929b..a1f586c 100644
--- a/src/syntax.zig
+++ b/src/syntax.zig
@@ -201,6 +201,11 @@ pub fn Token(comptime tag: Tag) type {
         pub fn extract(_: Tree, node: u32, _: void) @This() {
             return .{ .node = node };
         }
+
+        pub fn text(self: @This(), source: []const u8, tree: Tree) []const u8 {
+            const span = tree.token(self.node);
+            return source[span.start..span.end];
+        }
     };
 }
 

From 37a3e918b6f7176eefda5e141c543683fb75fd98 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Sun, 15 Oct 2023 12:57:08 +0200
Subject: [PATCH 02/29] use top-down name resolution algorithm

---
 src/analysis.zig | 304 +++++++++++++++++++----------------------------
 src/syntax.zig   |   3 +
 2 files changed, 125 insertions(+), 182 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index a85649a..fa01fcd 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -204,9 +204,9 @@ pub fn findDefinition(
     try visibleSymbols(arena, document, node, &symbols);
 
     for (symbols.items) |symbol| {
-        const parsed = try symbol.document.parseTree();
         if (std.mem.eql(u8, name, symbol.name())) {
             try references.append(symbol);
+            const parsed = try symbol.document.parseTree();
             if (!inFileRoot(parsed.tree, symbol.parent_declaration)) break;
         }
     }
@@ -322,6 +322,9 @@ pub const Scope = struct {
         shadowed: ?*Symbol = null,
     };
 
+    /// Call this before entering a new scope.
+    /// When a corresponding call to `end` is made, all symbols added within
+    /// this scope are no longer visible.
     pub fn begin(self: *@This()) !void {
         try self.active_scopes.append(self.allocator, self.next_scope);
         self.next_scope += 1;
@@ -380,7 +383,6 @@ pub fn visibleSymbols(
     start_node: u32,
     symbols: *std.ArrayList(Reference),
 ) !void {
-    _ = start_node;
     var scope = Scope{ .allocator = arena };
     try scope.begin();
     defer scope.end();
@@ -393,59 +395,153 @@ pub fn visibleSymbols(
         try documents.append(start_document);
         try findIncludedDocumentsRecursive(arena, &documents);
 
-        for (documents.items) |document| {
+        var documents_reverse = std.mem.reverseIterator(documents.items);
+        while (documents_reverse.next()) |document| {
             try collectGlobalSymbols(&scope, document);
         }
     }
 
+    try collectLocalSymbols(arena, &scope, start_document, start_node);
+
     try scope.getVisible(symbols);
 }
 
-pub fn collectGlobalSymbols(scope: *Scope, document: *Document) !void {
+fn collectLocalSymbols(
+    arena: std.mem.Allocator,
+    scope: *Scope,
+    document: *Document,
+    target_node: u32,
+) !void {
+    const parsed = try document.parseTree();
+    const tree = parsed.tree;
+
+    var path = try std.ArrayListUnmanaged(u32).initCapacity(arena, 8);
+    defer path.deinit(arena);
+
+    {
+        var child = target_node;
+        while (tree.parent(child)) |parent| : (child = parent) {
+            try path.append(arena, child);
+        }
+    }
+
+    var i = path.items.len - 1;
+    while (i >= 1) : (i -= 1) {
+        const parent = path.items[i];
+        const target_child = path.items[i - 1];
+
+        const children = tree.children(parent);
+        var current_child = children.start;
+        while (current_child < target_child) : (current_child += 1) {
+            if (syntax.ExternalDeclaration.tryExtract(tree, current_child)) |declaration| {
+                try collectDeclarationSymbols(scope, document, tree, declaration);
+                continue;
+            }
+            if (syntax.ParameterList.tryExtract(tree, current_child)) |parameters| {
+                var iterator = parameters.iterator();
+                while (iterator.next(tree)) |parameter| {
+                    const variable = parameter.get(.variable, tree) orelse continue;
+                    try registerVariables(scope, document, tree, .{ .one = variable }, parameter.node);
+                }
+                continue;
+            }
+            if (syntax.ConditionList.tryExtract(tree, current_child)) |condition_list| {
+                var statements = condition_list.iterator();
+                while (statements.next(tree)) |statement| {
+                    switch (statement) {
+                        .declaration => |declaration| {
+                            try collectDeclarationSymbols(
+                                scope,
+                                document,
+                                tree,
+                                .{ .variable = declaration },
+                            );
+                        },
+                    }
+                }
+                continue;
+            }
+        }
+    }
+}
+
+fn collectGlobalSymbols(scope: *Scope, document: *Document) !void {
     const parsed = try document.parseTree();
     const tree = parsed.tree;
 
     const children = tree.children(tree.root);
     for (children.start..children.end) |child| {
-        const global = syntax.AnyDeclaration.tryExtract(tree, @intCast(child)) orelse continue;
-        try collectSymbols(scope, document, tree, global);
+        const global = syntax.ExternalDeclaration.tryExtract(tree, @intCast(child)) orelse continue;
+        try collectDeclarationSymbols(scope, document, tree, global);
     }
 }
 
-fn collectSymbols(
+fn collectDeclarationSymbols(
     scope: *Scope,
     document: *Document,
     tree: parse.Tree,
-    any: syntax.AnyDeclaration,
+    external: syntax.ExternalDeclaration,
 ) !void {
-    switch (any) {
+    switch (external) {
         .function => |function| {
             const ident = function.get(.identifier, tree) orelse return;
-            try scope.add(ident.text(document.source(), tree), .{
-                .document = document,
-                .node = ident.node,
-                .parent_declaration = function.node,
-            });
+            try registerIdentifier(scope, document, tree, ident, function.node);
         },
         .variable => |declaration| {
-            const variables = declaration.get(.variables, tree) orelse return;
-            var iterator = variables.iterator();
-            while (iterator.next(tree)) |variable| {
-                const name = variable.get(.name, tree) orelse return;
-                const ident = name.getIdentifier(tree) orelse return;
-                try scope.add(ident.text(document.source(), tree), .{
-                    .document = document,
-                    .node = ident.node,
-                    .parent_declaration = declaration.node,
-                });
+            if (declaration.get(.specifier, tree)) |specifier| specifier: {
+                if (specifier != .struct_specifier) break :specifier;
+                const strukt = specifier.struct_specifier;
+                const ident = strukt.get(.name, tree) orelse break :specifier;
+                try registerIdentifier(scope, document, tree, ident, strukt.node);
             }
+
+            const variables = declaration.get(.variables, tree) orelse return;
+            try registerVariables(scope, document, tree, variables, declaration.node);
         },
-        else => {
-            std.log.warn("TODO: collect symbols from {s}", .{@tagName(any)});
+        .block => |block| {
+            if (block.get(.variable, tree)) |variable| {
+                try registerVariables(scope, document, tree, .{ .one = variable }, block.node);
+            } else {
+                const fields = block.get(.fields, tree) orelse return;
+                var field_iterator = fields.iterator();
+                while (field_iterator.next(tree)) |field| {
+                    const variables = field.get(.variables, tree) orelse continue;
+                    try registerVariables(scope, document, tree, variables, field.node);
+                }
+            }
         },
     }
 }
 
+fn registerIdentifier(
+    scope: *Scope,
+    document: *Document,
+    tree: Tree,
+    ident: syntax.Token(.identifier),
+    declaration: u32,
+) !void {
+    try scope.add(ident.text(document.source(), tree), .{
+        .document = document,
+        .node = ident.node,
+        .parent_declaration = declaration,
+    });
+}
+
+fn registerVariables(
+    scope: *Scope,
+    document: *Document,
+    tree: Tree,
+    variables: syntax.Variables,
+    declaration: u32,
+) !void {
+    var iterator = variables.iterator();
+    while (iterator.next(tree)) |variable| {
+        const name = variable.get(.name, tree) orelse return;
+        const ident = name.getIdentifier(tree) orelse return;
+        try registerIdentifier(scope, document, tree, ident, declaration);
+    }
+}
+
 /// Appends the set of documents which are visible (recursively) from any of the documents in the list.
 fn findIncludedDocumentsRecursive(
     arena: std.mem.Allocator,
@@ -505,162 +601,6 @@ fn findIncludedDocuments(
     }
 }
 
-/// Splits the list into two partitions: the first with all unique symbols, and the second with all duplicates.
-/// Returns the number of unique symbols.
-fn partitonUniqueSymbols(
-    allocator: std.mem.Allocator,
-    symbols: []Reference,
-    node_count: usize,
-) !usize {
-    // we want to keep the most recent symbol since its parent declaration will
-    // be higher up the tree, so begin by reversing the list (we keep the first
-    // unique value)
-    std.mem.reverse(Reference, symbols);
-
-    var visited = try std.DynamicBitSetUnmanaged.initEmpty(allocator, node_count);
-    defer visited.deinit(allocator);
-
-    var write: usize = 0;
-
-    for (symbols) |*symbol| {
-        if (visited.isSet(symbol.node)) continue;
-        visited.set(symbol.node);
-        std.mem.swap(Reference, &symbols[write], symbol);
-        write += 1;
-    }
-
-    // Restore the order the nodes were visited.
-    std.mem.reverse(Reference, symbols[0..write]);
-
-    return write;
-}
-
-fn visibleSymbolsTree(document: *Document, start_node: u32, symbols: *std.ArrayList(Reference)) !void {
-    const parse_tree = try document.parseTree();
-    const tree = parse_tree.tree;
-
-    // walk the tree upwards until we find the containing declaration
-    var current = start_node;
-    var previous = start_node;
-    while (true) : ({
-        previous = current;
-        current = tree.parent(current) orelse break;
-    }) {
-        const children = tree.children(current);
-
-        const tag = tree.tag(current);
-
-        var current_child = if (tag == .file)
-            children.end
-        else if (previous != current)
-            previous + 1
-        else
-            continue;
-
-        // search for the identifier among the children
-        while (current_child > children.start) {
-            current_child -= 1;
-            try findVisibleSymbols(
-                document,
-                tree,
-                current_child,
-                symbols,
-                .{ .check_children = tag != .file },
-            );
-        }
-    }
-}
-
-fn findVisibleSymbols(
-    document: *Document,
-    tree: Tree,
-    index: u32,
-    symbols: *std.ArrayList(Reference),
-    options: struct {
-        check_children: bool = true,
-        parent_declaration: ?u32 = null,
-    },
-) !void {
-    switch (tree.tag(index)) {
-        .function_declaration => {
-            if (syntax.FunctionDeclaration.tryExtract(tree, index)) |function| {
-                if (function.get(.identifier, tree)) |identifier| {
-                    try symbols.append(.{
-                        .document = document,
-                        .node = identifier.node,
-                        .parent_declaration = index,
-                    });
-                }
-            }
-
-            const children = tree.children(index);
-            var child = children.end;
-            while (child > children.start) {
-                child -= 1;
-                try findVisibleSymbols(document, tree, child, symbols, options);
-            }
-        },
-        .struct_specifier, .variable_declaration => {
-            const children = tree.children(index);
-            var child = children.end;
-            while (child > children.start) {
-                child -= 1;
-
-                if (syntax.VariableName.tryExtract(tree, child)) |name| {
-                    const identifier = name.getIdentifier(tree) orelse continue;
-                    try symbols.append(.{
-                        .document = document,
-                        .node = identifier.node,
-                        .parent_declaration = options.parent_declaration orelse index,
-                    });
-                    continue;
-                }
-
-                try findVisibleSymbols(document, tree, child, symbols, options);
-            }
-        },
-        .block, .statement => return,
-        else => |tag| {
-            if (tag.isToken()) return;
-
-            if (!options.check_children) {
-                if (tag == .parameter_list or tag == .field_declaration_list) {
-                    return;
-                }
-            }
-
-            const children = tree.children(index);
-            var child = children.end;
-            while (child > children.start) {
-                child -= 1;
-
-                var check_children = options.check_children;
-                if (syntax.BlockDeclaration.tryExtract(tree, child)) |block| {
-                    if (block.get(.variable, tree) == null) {
-                        // interface block without name:
-                        check_children = true;
-                    } else {
-                        check_children = false;
-                    }
-                }
-
-                try findVisibleSymbols(document, tree, child, symbols, .{
-                    .check_children = check_children,
-                    .parent_declaration = switch (tag) {
-                        .declaration,
-                        .parameter,
-                        .function_declaration,
-                        .block_declaration,
-                        .struct_specifier,
-                        => index,
-                        else => options.parent_declaration,
-                    },
-                });
-            }
-        },
-    }
-}
-
 fn nodeName(tree: Tree, node: u32, source: []const u8) ?[]const u8 {
     switch (tree.tag(node)) {
         .identifier => {
diff --git a/src/syntax.zig b/src/syntax.zig
index a1f586c..3cd3bfa 100644
--- a/src/syntax.zig
+++ b/src/syntax.zig
@@ -175,8 +175,11 @@ pub const Block = ListExtractor(.block, Token(.@"{"), Statement, Token(.@"}"));
 
 pub const Statement = union(enum) {
     pub usingnamespace UnionExtractorMixin(@This());
+    declaration: Declaration,
 };
 
+pub const ConditionList = ListExtractor(.condition_list, Token(.@"("), Statement, Token(.@")"));
+
 pub const Expression = Lazy("ExpressionUnion");
 
 pub const ExpressionUnion = union(enum) {

From 9018fc4753998f4247e812eb58e0236e044dcc9f Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Sun, 15 Oct 2023 21:27:07 +0200
Subject: [PATCH 03/29] fix struct completions for top-down semantics

---
 src/parse.zig | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/parse.zig b/src/parse.zig
index 8b838f4..3404335 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -618,6 +618,7 @@ fn externalDeclaration(p: *Parser) void {
     }
 
     typeQualifier(p);
+    const identifier_specifier = p.at(.identifier);
     if (p.atAny(type_specifier_first)) typeSpecifier(p);
 
     const m_field_list = p.open();
@@ -632,9 +633,7 @@ fn externalDeclaration(p: *Parser) void {
         return p.close(m, .block_declaration);
     }
 
-    if (p.eat(.@";")) return p.close(m, .qualifier_declaration);
-
-    if (p.at(.@",")) {
+    if (identifier_specifier and p.at(.@",")) {
         while (p.eat(.@",")) p.expect(.identifier);
         p.expect(.@";");
         return p.close(m, .qualifier_declaration);

From 29ec9a59d13af34a4783d666a0550dbebc74b84d Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Sun, 15 Oct 2023 22:01:02 +0200
Subject: [PATCH 04/29] allow hover on the current identifier

---
 src/analysis.zig | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index fa01fcd..d031442 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -418,9 +418,15 @@ fn collectLocalSymbols(
     var path = try std.ArrayListUnmanaged(u32).initCapacity(arena, 8);
     defer path.deinit(arena);
 
+    var closest_declaration: ?u32 = null;
+
     {
         var child = target_node;
         while (tree.parent(child)) |parent| : (child = parent) {
+            if (closest_declaration == null and syntax.AnyDeclaration.match(tree, parent) != null) {
+                closest_declaration = parent;
+            }
+
             try path.append(arena, child);
         }
     }
@@ -432,7 +438,7 @@ fn collectLocalSymbols(
 
         const children = tree.children(parent);
         var current_child = children.start;
-        while (current_child < target_child) : (current_child += 1) {
+        while (current_child < target_child or current_child == closest_declaration) : (current_child += 1) {
             if (syntax.ExternalDeclaration.tryExtract(tree, current_child)) |declaration| {
                 try collectDeclarationSymbols(scope, document, tree, declaration);
                 continue;

From f4b65f1e0db13d9599b28545e63545a0cceb9dee Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Sun, 15 Oct 2023 22:07:14 +0200
Subject: [PATCH 05/29] only hover words if there's an identifier under cursor

---
 src/Document.zig | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Document.zig b/src/Document.zig
index 2c1fd1e..7df8b16 100644
--- a/src/Document.zig
+++ b/src/Document.zig
@@ -85,11 +85,13 @@ pub fn nodeRange(self: *@This(), node: u32) !lsp.Range {
 
 pub fn wordUnderCursor(self: *@This(), cursor: lsp.Position) []const u8 {
     const offset = self.utf8FromPosition(cursor);
+    const bytes = self.contents.items;
+
+    if (!isIdentifierChar(bytes[offset])) return "";
 
     var start = offset;
     var end = offset;
 
-    const bytes = self.contents.items;
     while (start > 0 and isIdentifierChar(bytes[start - 1])) start -= 1;
     while (end < bytes.len and isIdentifierChar(bytes[end])) end += 1;
 

From b533e82721ebd26aca454219afa4c83dbfd4bbbb Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 19:39:27 +0200
Subject: [PATCH 06/29] escape newlines in comments

---
 justfile      |  2 ++
 src/parse.zig | 23 ++++++++++++++++++++---
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/justfile b/justfile
index d54c81f..412f816 100644
--- a/justfile
+++ b/justfile
@@ -18,6 +18,8 @@ watch *ARGS:
 
 test:
     zig build test --summary failures {{flags}}
+    just install
+    tests/run-all-tests.sh
 
 test-file *ARGS:
     zig test {{flags}} "$@"
diff --git a/src/parse.zig b/src/parse.zig
index 3404335..9f470df 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -385,6 +385,7 @@ pub const Parser = struct {
     allocator: std.mem.Allocator,
     tokenizer: Tokenizer,
     next: Node,
+    last_end: u32 = 0,
     fuel: u32 = max_fuel,
 
     deferred_error: ?Error = null,
@@ -463,6 +464,7 @@ pub const Parser = struct {
                     }
                 },
                 else => {
+                    self.last_end = self.next.span.end;
                     self.next = token;
                     self.fuel = max_fuel;
                     return;
@@ -509,7 +511,10 @@ pub const Parser = struct {
 
     fn emitError(self: *@This(), message: []const u8) void {
         self.emitDiagnostic(.{
-            .span = self.next.getToken() orelse unreachable,
+            .span = .{
+                .start = self.last_end,
+                .end = self.last_end,
+            },
             .message = message,
         });
 
@@ -520,7 +525,7 @@ pub const Parser = struct {
     fn advanceWithError(self: *@This(), message: []const u8) void {
         const m = self.open();
         self.emitDiagnostic(.{
-            .span = self.next.getToken() orelse unreachable,
+            .span = self.next.span,
             .message = message,
         });
         self.advance();
@@ -1415,7 +1420,19 @@ pub const Tokenizer = struct {
             },
             '/' => switch (getOrZero(text, i + 1)) {
                 '/' => {
-                    while (i < text.len and text[i] != '\n') i += 1;
+                    while (i < text.len) {
+                        switch (text[i]) {
+                            '\n' => break,
+                            '\\' => {
+                                if (getOrZero(text, i + 1) == '\n') {
+                                    i += 2;
+                                } else {
+                                    i += 1;
+                                }
+                            },
+                            else => i += 1,
+                        }
+                    }
                     return self.token(.comment, i);
                 },
                 '*' => {

From c445718b18abe9d3ae4d5fe01c13322730ae415c Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 19:54:20 +0200
Subject: [PATCH 07/29] handle line escapes in more places

---
 src/parse.zig | 70 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 32 deletions(-)

diff --git a/src/parse.zig b/src/parse.zig
index 9f470df..e96012e 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -1419,32 +1419,7 @@ pub const Tokenizer = struct {
                 else => return self.token(.@"*", i + 1),
             },
             '/' => switch (getOrZero(text, i + 1)) {
-                '/' => {
-                    while (i < text.len) {
-                        switch (text[i]) {
-                            '\n' => break,
-                            '\\' => {
-                                if (getOrZero(text, i + 1) == '\n') {
-                                    i += 2;
-                                } else {
-                                    i += 1;
-                                }
-                            },
-                            else => i += 1,
-                        }
-                    }
-                    return self.token(.comment, i);
-                },
-                '*' => {
-                    i += 2;
-                    while (i + 1 < text.len) : (i += 1) {
-                        if (std.mem.startsWith(u8, text[i..], "*/")) {
-                            i += 2;
-                            break;
-                        }
-                    }
-                    return self.token(.comment, i);
-                },
+                '/', '*' => return self.token(.comment, stripComment(text, i)),
                 '=' => return self.token(.@"/=", i + 2),
                 else => return self.token(.@"/", i + 1),
             },
@@ -1491,13 +1466,13 @@ pub const Tokenizer = struct {
             },
 
             '#' => {
-                while (i < N and text[i] != '\n') {
-                    if (text[i] == '\\') {
-                        i += 1;
-                        if (i < N) i += 1;
-                        continue;
+                while (i < N) {
+                    switch (text[i]) {
+                        '\n' => break,
+                        '\\' => i = @max(i + 1, stripLineEscape(text, i)),
+                        '/' => i = @max(i + 1, stripComment(text, i)),
+                        else => i += 1,
                     }
-                    i += 1;
                 }
 
                 return self.token(.preprocessor, i);
@@ -1577,6 +1552,37 @@ pub const Tokenizer = struct {
     }
 };
 
+fn stripLineEscape(text: []const u8, start: u32) u32 {
+    if (std.mem.startsWith(u8, text[start..], "\\\n")) return start + 2;
+    if (std.mem.startsWith(u8, text[start..], "\\\r\n")) return start + 3;
+    return start;
+}
+
+fn stripComment(text: []const u8, start: u32) u32 {
+    var i = start;
+
+    if (std.mem.startsWith(u8, text[i..], "//")) {
+        i += 2;
+        while (i < text.len) {
+            switch (text[i]) {
+                '\n' => break,
+                '\\' => i = @max(i + 1, stripLineEscape(text, i)),
+                else => i += 1,
+            }
+        }
+    } else if (std.mem.startsWith(u8, text[i..], "/*")) {
+        i += 2;
+        while (i < text.len) : (i += 1) {
+            if (std.mem.startsWith(u8, text[i..], "*/")) {
+                i += 2;
+                break;
+            }
+        }
+    }
+
+    return i;
+}
+
 fn stripPrefix(text: []const u8, prefix: []const u8) ?[]const u8 {
     return if (std.mem.startsWith(u8, text, prefix)) text[prefix.len..] else null;
 }

From 69c2ca2ba0f7cf33bd18407309792b223cb0818a Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 20:01:45 +0200
Subject: [PATCH 08/29] accept more integer suffixes

---
 src/parse.zig | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/parse.zig b/src/parse.zig
index e96012e..8c8f05b 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -1324,8 +1324,8 @@ pub const Tokenizer = struct {
                 // decimal/octal
                 while (i < N and std.ascii.isDigit(text[i])) i += 1;
 
-                if (i < N and (text[i] == 'u' or text[i] == 'U')) {
-                    // unsigned (cannot be float)
+                if (i < N and std.mem.indexOfScalar(u8, "uUsSlL", text[i]) != null) {
+                    // integer suffix (unsigned, short, long)
                     i += 1;
                     return self.token(.number, i);
                 }

From e121e70849a888da895adbb09c4541a3e24f980e Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 20:05:27 +0200
Subject: [PATCH 09/29] case-insensitive long-floats

---
 src/parse.zig | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/parse.zig b/src/parse.zig
index 8c8f05b..d91e51c 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -1350,9 +1350,7 @@ pub const Tokenizer = struct {
 
                 if (i < N and (text[i] == 'f' or text[i] == 'F')) {
                     i += 1;
-                } else if (i + 1 < N and (std.mem.startsWith(u8, text[i..], "lf") or
-                    std.mem.startsWith(u8, text[i..], "LF")))
-                {
+                } else if (std.ascii.startsWithIgnoreCase(text[i..], "lf")) {
                     i += 2;
                 }
 

From 8e6a4da495170493b28768b3210fa944d18f67e3 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 21:32:43 +0200
Subject: [PATCH 10/29] accept declarations in control flow conditions

---
 src/parse.zig | 68 ++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 20 deletions(-)

diff --git a/src/parse.zig b/src/parse.zig
index d91e51c..4b7552f 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -752,6 +752,7 @@ fn statement(p: *Parser) void {
     const m = p.open();
 
     switch (p.peek()) {
+        .@";" => return p.advance(),
         .@"{" => return block(p),
         .keyword_do => {
             p.advance();
@@ -771,7 +772,7 @@ fn statement(p: *Parser) void {
             {
                 const m_cond = p.open();
                 p.expect(.@"(");
-                expression(p);
+                condition(p);
                 p.expect(.@")");
                 p.close(m_cond, .condition_list);
             }
@@ -782,8 +783,8 @@ fn statement(p: *Parser) void {
             {
                 const m_cond = p.open();
                 p.expect(.@"(");
-                if (!p.eat(.@";")) statement(p);
-                if (!p.eat(.@";")) statement(p);
+                if (!p.eat(.@";")) conditionStatement(p);
+                if (!p.eat(.@";")) conditionStatement(p);
                 _ = expressionOpt(p);
                 p.expect(.@")");
                 p.close(m_cond, .condition_list);
@@ -797,7 +798,7 @@ fn statement(p: *Parser) void {
                 {
                     const m_cond = p.open();
                     p.expect(.@"(");
-                    expression(p);
+                    condition(p);
                     p.expect(.@")");
                     p.close(m_cond, .condition_list);
                 }
@@ -820,7 +821,7 @@ fn statement(p: *Parser) void {
             {
                 const m_cond = p.open();
                 p.expect(.@"(");
-                expression(p);
+                condition(p);
                 p.expect(.@")");
                 p.close(m_cond, .condition_list);
             }
@@ -847,27 +848,54 @@ fn statement(p: *Parser) void {
             p.expect(.@";");
         },
         else => {
-            var is_decl = false;
-
-            if (p.atAny(type_qualifier_first)) {
-                typeQualifier(p);
-                typeSpecifier(p);
-                is_decl = true;
-            } else {
-                if (!expressionOpt(p)) p.emitError("expected a statement");
-            }
-
-            if (variableDeclarationList(p) > 0) is_decl = true;
-
+            const kind = simpleStatement(p);
             p.expect(.@";");
-
-            if (is_decl) return p.close(m, .declaration);
+            if (kind == .declaration) return p.close(m, .declaration);
         },
     }
 
     p.close(m, .statement);
 }
 
+fn conditionStatement(p: *Parser) void {
+    const m = p.open();
+    const kind = simpleStatement(p);
+    p.expect(.@";");
+    if (kind == .declaration) return p.close(m, .declaration);
+}
+
+fn condition(p: *Parser) void {
+    const m = p.open();
+    switch (simpleStatement(p)) {
+        .declaration => p.close(m, .declaration),
+        .expression => {},
+    }
+}
+
+fn simpleStatement(p: *Parser) enum { declaration, expression } {
+    var is_decl = false;
+    var has_specifier = false;
+
+    if (p.atAny(type_qualifier_first)) {
+        typeQualifier(p);
+        typeSpecifier(p);
+        is_decl = true;
+        has_specifier = true;
+    } else {
+        if (!expressionOpt(p)) p.emitError("expected a statement");
+
+        const tags = p.stack.items(.tag);
+        has_specifier = switch (tags[tags.len - 1]) {
+            .array_specifier, .struct_specifier, .identifier => true,
+            else => false,
+        };
+    }
+
+    if (has_specifier and variableDeclarationList(p) > 0) is_decl = true;
+
+    return if (is_decl) .declaration else .expression;
+}
+
 fn variableDeclarationList(p: *Parser) u32 {
     const m_vars = p.open();
     var var_count: u32 = 0;
@@ -887,7 +915,7 @@ fn variableDeclaration(p: *Parser) void {
 fn variableDeclarationSuffix(p: *Parser, m_var: Parser.Mark) void {
     if (p.at(.@"[")) arraySpecifier(p, m_var);
     if (p.eat(.@"=")) initializer(p);
-    if (!p.at(.@";")) p.expect(.@",");
+    if (!p.at(.@";") and !p.at(.@")")) p.expect(.@",");
     p.close(m_var, .variable_declaration);
 }
 

From 614c9a184399273b3e8860f65f53c36b14b5bbb9 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 21:39:05 +0200
Subject: [PATCH 11/29] accept any number suffix

---
 src/parse.zig | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/parse.zig b/src/parse.zig
index 4b7552f..aaea0cd 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -1376,11 +1376,8 @@ pub const Tokenizer = struct {
                     while (i < N and std.ascii.isDigit(text[i])) i += 1;
                 }
 
-                if (i < N and (text[i] == 'f' or text[i] == 'F')) {
-                    i += 1;
-                } else if (std.ascii.startsWithIgnoreCase(text[i..], "lf")) {
-                    i += 2;
-                }
+                // type suffix (we just accept anything here to be as permissive as possible)
+                while (i < N and isIdentifierChar(text[i])) i += 1;
 
                 return self.token(.number, i);
             },

From 089d46fb6c5eff28200e52699947fba263ada677 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 21:56:17 +0200
Subject: [PATCH 12/29] detect macro expansion at declaration level

---
 src/parse.zig | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/src/parse.zig b/src/parse.zig
index aaea0cd..619c7e0 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -587,6 +587,11 @@ pub const Parser = struct {
             .span = range,
         }));
     }
+
+    fn lastTag(self: *const @This()) Tag {
+        const tags = self.stack.items(.tag);
+        return tags[tags.len - 1];
+    }
 };
 
 pub fn parseFile(p: *Parser) void {
@@ -623,9 +628,29 @@ fn externalDeclaration(p: *Parser) void {
     }
 
     typeQualifier(p);
+
     const identifier_specifier = p.at(.identifier);
     if (p.atAny(type_specifier_first)) typeSpecifier(p);
 
+    if (identifier_specifier and p.lastTag() == .identifier and p.at(.@"(")) {
+        // looks like macro expansion
+        var level: u32 = 0;
+        while (true) {
+            const tag = p.peek();
+            defer p.advance();
+            switch (tag) {
+                .@"(" => level += 1,
+                .@")" => {
+                    level -= 1;
+                    if (level == 0) break;
+                },
+                .eof => break,
+                else => {},
+            }
+        }
+        return p.close(m, .call);
+    }
+
     const m_field_list = p.open();
     if (p.eat(.@"{")) {
         while (p.atAny(struct_field_declaration_first)) {
@@ -883,9 +908,7 @@ fn simpleStatement(p: *Parser) enum { declaration, expression } {
         has_specifier = true;
     } else {
         if (!expressionOpt(p)) p.emitError("expected a statement");
-
-        const tags = p.stack.items(.tag);
-        has_specifier = switch (tags[tags.len - 1]) {
+        has_specifier = switch (p.lastTag()) {
             .array_specifier, .struct_specifier, .identifier => true,
             else => false,
         };
@@ -1352,12 +1375,6 @@ pub const Tokenizer = struct {
                 // decimal/octal
                 while (i < N and std.ascii.isDigit(text[i])) i += 1;
 
-                if (i < N and std.mem.indexOfScalar(u8, "uUsSlL", text[i]) != null) {
-                    // integer suffix (unsigned, short, long)
-                    i += 1;
-                    return self.token(.number, i);
-                }
-
                 if (i < N and text[i] == '.') {
                     // fractional part
                     i += 1;

From 21e4e13c0c14ce7a14af3507a8ad0ac42c08046c Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Mon, 16 Oct 2023 23:41:09 +0200
Subject: [PATCH 13/29] hotfix: local-scope shadowing + goto-def for fields

---
 src/analysis.zig | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index d031442..4f5e1e8 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -192,7 +192,7 @@ pub fn findDefinition(
     document: *Document,
     node: u32,
     references: *std.ArrayList(Reference),
-) !void {
+) error{OutOfMemory}!void {
     const parse_tree = try document.parseTree();
     const tree = parse_tree.tree;
 
@@ -201,7 +201,10 @@ pub fn findDefinition(
     var symbols = std.ArrayList(Reference).init(arena);
     defer symbols.deinit();
 
-    try visibleSymbols(arena, document, node, &symbols);
+    try visibleFields(arena, document, node, &symbols);
+    if (symbols.items.len == 0) {
+        try visibleSymbols(arena, document, node, &symbols);
+    }
 
     for (symbols.items) |symbol| {
         if (std.mem.eql(u8, name, symbol.name())) {
@@ -358,17 +361,16 @@ pub const Scope = struct {
     }
 
     pub fn isActive(self: *const @This(), scope: ScopeId) bool {
-        for (self.active_scopes.items) |active| {
-            if (scope == active) return true;
-        }
-        return false;
+        return std.mem.lastIndexOfScalar(ScopeId, self.active_scopes.items, scope) != null;
     }
 
     pub fn getVisible(self: *const @This(), symbols: *std.ArrayList(Reference)) !void {
         try symbols.ensureUnusedCapacity(self.symbols.count());
         for (self.symbols.values()) |*value| {
+            const first_scope = value.scope;
             var current: ?*Symbol = value;
             while (current) |symbol| : (current = symbol.shadowed) {
+                if (symbol.scope != first_scope) break;
                 if (!self.isActive(symbol.scope)) continue;
                 try symbols.append(symbol.reference);
             }
@@ -401,9 +403,12 @@ pub fn visibleSymbols(
         }
     }
 
-    try collectLocalSymbols(arena, &scope, start_document, start_node);
-
-    try scope.getVisible(symbols);
+    {
+        try scope.begin();
+        defer scope.end();
+        try collectLocalSymbols(arena, &scope, start_document, start_node);
+        try scope.getVisible(symbols);
+    }
 }
 
 fn collectLocalSymbols(

From f1e2cf6e4814c116492cf7ed6e520c7c3b1320dd Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 07:47:31 +0200
Subject: [PATCH 14/29] Allow any error in recursive function

---
 src/analysis.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index 4f5e1e8..cb89493 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -192,7 +192,7 @@ pub fn findDefinition(
     document: *Document,
     node: u32,
     references: *std.ArrayList(Reference),
-) error{OutOfMemory}!void {
+) anyerror!void {
     const parse_tree = try document.parseTree();
     const tree = parse_tree.tree;
 

From 559ec4de3d78457b43c7f7eea0856bb70fff9f4f Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 08:47:39 +0200
Subject: [PATCH 15/29] enable parallel release builds

---
 build.zig  | 97 +++++++++++++++++++++++++++++++++++++++---------------
 justfile   |  2 +-
 release.sh | 21 +++++-------
 3 files changed, 79 insertions(+), 41 deletions(-)

diff --git a/build.zig b/build.zig
index ec1be27..f9045b0 100644
--- a/build.zig
+++ b/build.zig
@@ -4,40 +4,83 @@ pub fn build(b: *std.Build) !void {
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
-    const exe = b.addExecutable(.{
-        .name = "glsl_analyzer",
-        .root_source_file = .{ .path = "src/main.zig" },
-        .target = target,
-        .optimize = optimize,
-    });
-    try attachModules(exe);
-    b.installArtifact(exe);
+    // Executable
+    {
+        const exe = try addExecutable(b, .{ .target = target, .optimize = optimize });
+        b.installArtifact(exe);
 
-    const run_cmd = b.addRunArtifact(exe);
-    run_cmd.step.dependOn(b.getInstallStep());
+        const run_cmd = b.addRunArtifact(exe);
+        run_cmd.step.dependOn(b.getInstallStep());
 
-    if (b.args) |args| {
-        run_cmd.addArgs(args);
-    }
+        if (b.args) |args| {
+            run_cmd.addArgs(args);
+        }
 
-    const run_step = b.step("run", "Run the app");
-    run_step.dependOn(&run_cmd.step);
+        const run_step = b.step("run", "Run the app");
+        run_step.dependOn(&run_cmd.step);
+    }
 
-    const unit_tests = b.addTest(.{
-        .name = "unit-tests",
-        .root_source_file = .{ .path = "src/main.zig" },
-        .target = target,
-        .optimize = optimize,
-    });
-    try attachModules(unit_tests);
+    // Tests
+    {
+        const unit_tests = b.addTest(.{
+            .name = "unit-tests",
+            .root_source_file = .{ .path = "src/main.zig" },
+            .target = target,
+            .optimize = optimize,
+        });
+        try attachModules(unit_tests);
+
+        if (b.option(bool, "install-tests", "Install the unit tests in the `bin` folder") orelse false) {
+            b.installArtifact(unit_tests);
+        }
+
+        const run_unit_tests = b.addRunArtifact(unit_tests);
+        const test_step = b.step("test", "Run unit tests");
+        test_step.dependOn(&run_unit_tests.step);
+    }
 
-    if (b.option(bool, "install-tests", "Install the unit tests in the `bin` folder") orelse false) {
-        b.installArtifact(unit_tests);
+    // Release
+    {
+        const target_triples = [_][]const u8{
+            "x86_64-linux-musl",
+            "aarch64-linux-musl",
+            "x86_64-macos",
+            "aarch64-macos",
+            "x86_64-windows",
+            "aarch64-windows",
+        };
+        const release_step = b.step("release", "Produce executables for targeted platforms");
+
+        for (&target_triples) |triple| {
+            const release_target = try std.zig.CrossTarget.parse(.{
+                .arch_os_abi = triple,
+                .cpu_features = "baseline",
+            });
+
+            const exe = try addExecutable(b, .{ .target = release_target, .optimize = optimize });
+            const install = b.addInstallArtifact(exe, .{
+                .dest_dir = .{ .override = .{
+                    .custom = b.pathJoin(&.{ triple, "bin" }),
+                } },
+            });
+
+            release_step.dependOn(&install.step);
+        }
     }
+}
 
-    const run_unit_tests = b.addRunArtifact(unit_tests);
-    const test_step = b.step("test", "Run unit tests");
-    test_step.dependOn(&run_unit_tests.step);
+fn addExecutable(b: *std.Build, options: struct {
+    target: std.zig.CrossTarget,
+    optimize: std.builtin.OptimizeMode,
+}) !*std.Build.CompileStep {
+    const exe = b.addExecutable(.{
+        .name = "glsl_analyzer",
+        .root_source_file = .{ .path = "src/main.zig" },
+        .target = options.target,
+        .optimize = options.optimize,
+    });
+    try attachModules(exe);
+    return exe;
 }
 
 fn attachModules(step: *std.Build.CompileStep) !void {
diff --git a/justfile b/justfile
index 412f816..0743ca0 100644
--- a/justfile
+++ b/justfile
@@ -41,4 +41,4 @@ generate-spec:
     cd spec && just
 
 release:
-    ./release.sh
+    zig build release {{flags}}
diff --git a/release.sh b/release.sh
index e017a87..24c4e5f 100755
--- a/release.sh
+++ b/release.sh
@@ -2,20 +2,15 @@
 
 set -e
 
-targets=(
-    x86_64-linux-musl
-    aarch64-linux-musl
-    x86_64-macos
-    aarch64-macos
-    x86_64-windows
-    aarch64-windows
-)
+rm -rf "zig-out/release"
+rm -rf "zig-out/archives"
+
+zig build release -Doptimize=ReleaseSafe --prefix "zig-out/release" --verbose
 
 mkdir -p "zig-out/archives"
 
-for target in ${targets[@]}; do
-    echo "building $target..."
-    mkdir -p "zig-out/$target"
-    zig build -Dtarget=$target -Doptimize=ReleaseSafe --prefix "zig-out/$target"
-    (cd "zig-out/$target/" && zip -r "../archives/$target.zip" *)
+for target_path in zig-out/release/*; do
+    target=$(basename "$target_path")
+    echo "archiving $target..."
+    (cd "zig-out/release/$target/" && zip -r "../../archives/$target.zip" *)
 done

From bc7079b388e2f139b292a1539d3cb7a621455b21 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 18:34:21 +0200
Subject: [PATCH 16/29] build & test PRs in CI

---
 .github/workflows/pull-request.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 .github/workflows/pull-request.yml

diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
new file mode 100644
index 0000000..ea6d1dd
--- /dev/null
+++ b/.github/workflows/pull-request.yml
@@ -0,0 +1,22 @@
+name: "Pull Request CI"
+on:
+  pull_request:
+    types: [opened, synchronize]
+permissions:
+  contents: write
+jobs:
+  build-artifacts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Zig
+        uses: goto-bus-stop/setup-zig@v2.1.1
+        with:
+          version: 0.12.0-dev.790+ad6f8e3a5
+      - name: Build
+        run: zig build release -Doptimize=Debug
+      - name: Test
+        run: zig build test
+          
+
+

From 2bf756d6436321c93ded3496c07c6a62b6737b1d Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 18:39:58 +0200
Subject: [PATCH 17/29] allow `git describe` even in shallow clone

---
 build.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.zig b/build.zig
index f9045b0..08463b7 100644
--- a/build.zig
+++ b/build.zig
@@ -97,7 +97,7 @@ fn attachModules(step: *std.Build.CompileStep) !void {
         &.{b.build_root.path orelse "."},
     );
     options.addOption([]const u8, "build_root", build_root_path);
-    options.addOption([]const u8, "version", b.exec(&.{ "git", "describe", "--tags" }));
+    options.addOption([]const u8, "version", b.exec(&.{ "git", "describe", "--tags", "--always" }));
     step.addOptions("build_options", options);
 }
 

From 77427fc38715fb58498788c1f58c6d6cf4793ac0 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 18:57:09 +0200
Subject: [PATCH 18/29] avoid possible infinite recursion

---
 src/analysis.zig | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index cb89493..df8fe59 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -226,20 +226,20 @@ fn inFileRoot(tree: Tree, node: u32) bool {
 pub fn visibleFields(
     arena: std.mem.Allocator,
     document: *Document,
-    node: u32,
+    start_node: u32,
     symbols: *std.ArrayList(Reference),
 ) !void {
     const name = blk: {
         const parsed = try document.parseTree();
-        const tag = parsed.tree.tag(node);
+        const tag = parsed.tree.tag(start_node);
         if (tag != .identifier and tag != .@".") return;
 
-        const parent = parsed.tree.parent(node) orelse return;
+        const parent = parsed.tree.parent(start_node) orelse return;
         const parent_tag = parsed.tree.tag(parent);
         if (parent_tag != .selection) return;
 
         const children = parsed.tree.children(parent);
-        if (node == children.start) return;
+        if (start_node == children.start) return;
 
         var first = children.start;
         while (parsed.tree.tag(first).isSyntax()) {
@@ -250,6 +250,11 @@ pub fn visibleFields(
         break :blk first;
     };
 
+    if (start_node == name) {
+        // possible infinite loop if we are coming from `findDefinition`
+        return;
+    }
+
     var name_definitions = std.ArrayList(Reference).init(arena);
     try findDefinition(arena, document, name, &name_definitions);
     if (name_definitions.items.len == 0) return;
@@ -284,6 +289,7 @@ pub fn visibleFields(
                     },
                     else => {
                         const identifier = specifier.underlyingName(tree) orelse continue;
+                        if (identifier.node == start_node) continue;
                         try findDefinition(arena, reference.document, identifier.node, &references);
                     },
                 }

From 99c495393d86c65d8bf47d798395850e472d83d9 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 19:35:03 +0200
Subject: [PATCH 19/29] implement local variable shadowing

---
 src/analysis.zig | 197 +++++++++++++++++++++++++++--------------------
 1 file changed, 112 insertions(+), 85 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index df8fe59..7e5e171 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -168,12 +168,10 @@ fn expectTypeFormat(source: []const u8, types: []const []const u8) !void {
     if (cursors.count() != types.len) return error.InvalidCursorCount;
 
     for (types, cursors.values()) |expected, cursor| {
-        if (cursor.usages.len != 0) return error.DuplicateCursor;
-
         var references = std.ArrayList(Reference).init(allocator);
         defer references.deinit();
 
-        try findDefinition(allocator, document, cursor.definition, &references);
+        try findDefinition(allocator, document, cursor.node, &references);
         if (references.items.len != 1) return error.InvalidReference;
         const ref = references.items[0];
 
@@ -373,12 +371,19 @@ pub const Scope = struct {
     pub fn getVisible(self: *const @This(), symbols: *std.ArrayList(Reference)) !void {
         try symbols.ensureUnusedCapacity(self.symbols.count());
         for (self.symbols.values()) |*value| {
-            const first_scope = value.scope;
+            var first_scope: ?u32 = null;
             var current: ?*Symbol = value;
             while (current) |symbol| : (current = symbol.shadowed) {
-                if (symbol.scope != first_scope) break;
                 if (!self.isActive(symbol.scope)) continue;
+
+                // symbols from parent scopes are shadowed by the first scope
+                first_scope = first_scope orelse symbol.scope;
+                if (symbol.scope != first_scope) break;
+
                 try symbols.append(symbol.reference);
+
+                // only symbols in global scope can be overloaded
+                if (symbol.scope != 0) break;
             }
         }
     }
@@ -637,75 +642,107 @@ fn nodeName(tree: Tree, node: u32, source: []const u8) ?[]const u8 {
 }
 
 test "find definition local variable" {
-    try expectDefinitionIsFound(
+    try expectDefinition(
         \\void main() {
-        \\    int /*1*/x = 1;
+        \\    int /*2*/x = 1;
         \\    /*1*/x += 2;
         \\}
-    );
-    try expectDefinitionIsFound(
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+    });
+    try expectDefinition(
         \\void main() {
-        \\    for (int /*1*/i = 0; i < 10; i++) {
+        \\    for (int /*2*/i = 0; i < 10; i++) {
         \\         /*1*/i += 1;
         \\    }
         \\}
-    );
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+    });
+    try expectDefinition(
+        \\void main() {
+        \\    int /*3*/foo;
+        \\    {
+        \\        float /*2*/foo;
+        \\        /*1*/foo;
+        \\    }
+        \\}
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+        .{ .source = "/*1*/", .target = "/*3*/", .should_exist = false },
+    });
 }
 
 test "find definition parameter" {
-    try expectDefinitionIsFound(
-        \\int bar(int /*1*/x) {
+    try expectDefinition(
+        \\int bar(int /*2*/x) {
         \\    return /*1*/x;
         \\}
-    );
-    try expectDefinitionIsNotFound(
-        \\int foo(int /*1*/x) { return x; }
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+    });
+    try expectDefinition(
+        \\int foo(int /*2*/x) { return x; }
         \\int bar() {
         \\    return /*1*/x;
         \\}
-    );
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = false },
+    });
 }
 
 test "find definition function" {
-    try expectDefinitionIsFound(
-        \\void /*1*/foo() {}
+    try expectDefinition(
+        \\void /*3*/foo(int x) {}
+        \\void /*2*/foo() {}
         \\void main() {
         \\    /*1*/foo();
         \\}
-    );
-    try expectDefinitionIsFound(
-        \\void foo() {}
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+        .{ .source = "/*1*/", .target = "/*3*/", .should_exist = true },
+    });
+    try expectDefinition(
+        \\void /*3*/foo() {}
         \\void main() {
-        \\    int /*1*/foo = 123;
+        \\    int /*2*/foo = 123;
         \\    /*1*/foo();
         \\}
-    );
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+        .{ .source = "/*1*/", .target = "/*3*/", .should_exist = false },
+    });
 }
 
 test "find definition global" {
-    try expectDefinitionIsFound(
-        \\layout(location = 1) uniform vec4 /*1*/color;
+    try expectDefinition(
+        \\layout(location = 1) uniform vec4 /*2*/color;
         \\void main() {
         \\    /*1*/color;
         \\}
-    );
-    try expectDefinitionIsFound(
-        \\layout(location = 1) uniform MyBlock { vec4 color; } /*1*/my_block;
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+    });
+    try expectDefinition(
+        \\layout(location = 1) uniform MyBlock { vec4 /*4*/color; } /*2*/my_block;
         \\void main() {
-        \\    color;
+        \\    /*3*/color;
         \\    /*1*/my_block;
         \\}
-    );
-    try expectDefinitionIsNotFound(
-        \\layout(location = 1) uniform MyBlock { vec4 /*1*/color; } my_block;
-        \\void main() {
-        \\    /*1*/color;
-        \\    my_block;
-        \\}
-    );
+    , &.{
+        .{ .source = "/*1*/", .target = "/*2*/", .should_exist = true },
+        .{ .source = "/*3*/", .target = "/*4*/", .should_exist = false },
+    });
 }
 
-fn expectDefinitionIsFound(source: []const u8) !void {
+fn expectDefinition(
+    source: []const u8,
+    cases: []const struct {
+        source: []const u8,
+        target: []const u8,
+        should_exist: bool,
+    },
+) !void {
     var workspace = try Workspace.init(std.testing.allocator);
     defer workspace.deinit();
 
@@ -718,73 +755,63 @@ fn expectDefinitionIsFound(source: []const u8) !void {
     var cursors = try findCursors(document);
     defer cursors.deinit();
 
-    for (cursors.values()) |cursor| {
-        for (cursor.usages.slice()) |usage| {
-            var references = std.ArrayList(Reference).init(workspace.allocator);
-            defer references.deinit();
-            try findDefinition(arena.allocator(), document, usage, &references);
-            if (references.items.len == 0) return error.ReferenceNotFound;
-            if (references.items.len > 1) return error.MultipleDefinitions;
-            const ref = references.items[0];
-            try std.testing.expectEqual(document, ref.document);
-            try std.testing.expectEqual(cursor.definition, ref.node);
-        }
-    }
-}
-
-fn expectDefinitionIsNotFound(source: []const u8) !void {
-    var workspace = try Workspace.init(std.testing.allocator);
-    defer workspace.deinit();
+    for (cases) |case| {
+        const usage = cursors.get(case.source) orelse std.debug.panic("invalid cursor: {s}", .{case.source});
+        const definition = cursors.get(case.target) orelse std.debug.panic("invalid cursor: {s}", .{case.source});
 
-    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
-    defer arena.deinit();
+        var references = std.ArrayList(Reference).init(workspace.allocator);
+        defer references.deinit();
+        try findDefinition(arena.allocator(), document, usage.node, &references);
 
-    const document = try workspace.getOrCreateDocument(.{ .uri = "file://test.glsl", .version = 0 });
-    try document.replaceAll(source);
+        var found_definition = false;
+        for (references.items) |reference| {
+            if (reference.document == document and reference.node == definition.node) {
+                found_definition = true;
+                break;
+            }
+        }
 
-    var cursors = try findCursors(document);
-    defer cursors.deinit();
+        if (case.should_exist and !found_definition) {
+            std.log.err(
+                "{s} did not find {s}:\n================\n{s}\n================",
+                .{ case.source, case.target, source },
+            );
+        }
 
-    for (cursors.values()) |cursor| {
-        for (cursor.usages.slice()) |usage| {
-            var references = std.ArrayList(Reference).init(workspace.allocator);
-            defer references.deinit();
-            try findDefinition(arena.allocator(), document, usage, &references);
-            if (references.items.len != 0) {
-                const ref = references.items[0];
-                std.debug.print("found unexpected reference: {s}:{}\n", .{ ref.document.path, ref.node });
-                return error.FoundUnexpectedReference;
-            }
+        if (!case.should_exist and found_definition) {
+            std.log.err(
+                "{s} did find {s}:\n================\n{s}\n================",
+                .{ case.source, case.target, source },
+            );
         }
     }
 }
 
 const Cursor = struct {
-    definition: u32,
-    usages: std.BoundedArray(u32, 4) = .{},
+    node: u32,
 };
 
 fn findCursors(document: *Document) !std.StringArrayHashMap(Cursor) {
     const parsed = try document.parseTree();
     const tree = &parsed.tree;
 
-    var cursors = std.StringArrayHashMap(Cursor).init(document.workspace.allocator);
+    var cursors = std.StringArrayHashMap(Cursor).init(std.testing.allocator);
     errdefer cursors.deinit();
 
-    for (0..tree.nodes.len) |index| {
-        const node = tree.nodes.get(index);
-        const token = node.getToken() orelse continue;
-        for (parsed.ignored) |cursor| {
+    for (parsed.ignored) |cursor| {
+        for (tree.nodes.items(.span), tree.nodes.items(.tag), 0..) |token, tag, index| {
+            if (tag.isSyntax()) continue;
             if (cursor.end == token.start) {
-                const result = try cursors.getOrPut(
+                try cursors.putNoClobber(
                     document.source()[cursor.start..cursor.end],
+                    .{ .node = @intCast(index) },
                 );
-                if (result.found_existing) {
-                    try result.value_ptr.usages.append(@intCast(index));
-                } else {
-                    result.value_ptr.* = .{ .definition = @intCast(index) };
-                }
+                break;
             }
+        } else {
+            std.debug.panic("cursor not found: \"{}\"", .{
+                std.zig.fmtEscapes(document.source()[cursor.start..cursor.end]),
+            });
         }
     }
 

From 1cff5552ff3f40a4b47121f155c431ab31933214 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 19:51:33 +0200
Subject: [PATCH 20/29] identify hover word by querying parse tree

---
 src/Document.zig | 19 ++-----------------
 src/main.zig     | 41 ++++++++++++++++++++++++++---------------
 2 files changed, 28 insertions(+), 32 deletions(-)

diff --git a/src/Document.zig b/src/Document.zig
index 7df8b16..a505233 100644
--- a/src/Document.zig
+++ b/src/Document.zig
@@ -83,23 +83,8 @@ pub fn nodeRange(self: *@This(), node: u32) !lsp.Range {
     };
 }
 
-pub fn wordUnderCursor(self: *@This(), cursor: lsp.Position) []const u8 {
-    const offset = self.utf8FromPosition(cursor);
-    const bytes = self.contents.items;
-
-    if (!isIdentifierChar(bytes[offset])) return "";
-
-    var start = offset;
-    var end = offset;
-
-    while (start > 0 and isIdentifierChar(bytes[start - 1])) start -= 1;
-    while (end < bytes.len and isIdentifierChar(bytes[end])) end += 1;
-
-    return bytes[start..end];
-}
-
 /// Return the node right under the cursor.
-pub fn nodeUnderCursor(self: *@This(), cursor: lsp.Position) !?u32 {
+pub fn tokenUnderCursor(self: *@This(), cursor: lsp.Position) !?u32 {
     const offset = self.utf8FromPosition(cursor);
     const parsed = try self.parseTree();
     const tree = parsed.tree;
@@ -114,7 +99,7 @@ pub fn nodeUnderCursor(self: *@This(), cursor: lsp.Position) !?u32 {
 }
 
 /// Return the node closest to left of the cursor.
-pub fn nodeBeforeCursor(self: *@This(), cursor: lsp.Position) !?u32 {
+pub fn tokenBeforeCursor(self: *@This(), cursor: lsp.Position) !?u32 {
     const offset = self.utf8FromPosition(cursor);
     const parsed = try self.parseTree();
     const tree = parsed.tree;
diff --git a/src/main.zig b/src/main.zig
index b65e73c..56d7277 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -545,10 +545,12 @@ pub const Dispatch = struct {
         var symbol_arena = std.heap.ArenaAllocator.init(state.allocator);
         defer symbol_arena.deinit();
 
-        try completionsAtPosition(
+        const token = try document.tokenBeforeCursor(params.value.position);
+
+        try completionsAtToken(
             state,
             document,
-            params.value.position,
+            token,
             &completions,
             symbol_arena.allocator(),
             .{ .ignore_current = true },
@@ -557,28 +559,28 @@ pub const Dispatch = struct {
         try state.success(request.id, completions.items);
     }
 
-    fn completionsAtPosition(
+    fn completionsAtToken(
         state: *State,
         document: *Workspace.Document,
-        position: lsp.Position,
+        start_token: ?u32,
         completions: *std.ArrayList(lsp.CompletionItem),
         arena: std.mem.Allocator,
         options: struct { ignore_current: bool },
     ) !void {
         var has_fields = false;
 
-        if (try document.nodeBeforeCursor(position)) |node| {
-            var symbols = std.ArrayList(analysis.Reference).init(arena);
+        var symbols = std.ArrayList(analysis.Reference).init(arena);
 
-            try analysis.visibleFields(arena, document, node, &symbols);
+        if (start_token) |token| {
+            try analysis.visibleFields(arena, document, token, &symbols);
             has_fields = symbols.items.len != 0;
 
-            if (!has_fields) try analysis.visibleSymbols(arena, document, node, &symbols);
+            if (!has_fields) try analysis.visibleSymbols(arena, document, token, &symbols);
 
             try completions.ensureUnusedCapacity(symbols.items.len);
 
             for (symbols.items) |symbol| {
-                if (options.ignore_current and symbol.document == document and symbol.node == node) {
+                if (options.ignore_current and symbol.document == document and symbol.node == token) {
                     continue;
                 }
 
@@ -635,9 +637,18 @@ pub const Dispatch = struct {
         std.log.debug("hover: {} {s}", .{ params.value.position, params.value.textDocument.uri });
 
         const document = try getDocumentOrFail(state, request, params.value.textDocument);
+        const parsed = try document.parseTree();
 
-        const word = document.wordUnderCursor(params.value.position);
-        std.log.debug("hover word: '{'}'", .{std.zig.fmtEscapes(word)});
+        const token = try document.tokenUnderCursor(params.value.position) orelse {
+            return state.success(request.id, null);
+        };
+
+        if (parsed.tree.tag(token) != .identifier) {
+            return state.success(request.id, null);
+        }
+
+        const token_span = parsed.tree.token(token);
+        const token_text = document.source()[token_span.start..token_span.end];
 
         var completions = std.ArrayList(lsp.CompletionItem).init(state.allocator);
         defer completions.deinit();
@@ -645,10 +656,10 @@ pub const Dispatch = struct {
         var symbol_arena = std.heap.ArenaAllocator.init(state.allocator);
         defer symbol_arena.deinit();
 
-        try completionsAtPosition(
+        try completionsAtToken(
             state,
             document,
-            params.value.position,
+            token,
             &completions,
             symbol_arena.allocator(),
             .{ .ignore_current = false },
@@ -658,7 +669,7 @@ pub const Dispatch = struct {
         defer text.deinit();
 
         for (completions.items) |*completion| {
-            if (std.mem.eql(u8, completion.label, word)) {
+            if (std.mem.eql(u8, completion.label, token_text)) {
                 if (text.items.len != 0) {
                     try text.appendSlice("\n\n---\n\n");
                 }
@@ -732,7 +743,7 @@ pub const Dispatch = struct {
         });
 
         const document = try state.workspace.getOrLoadDocument(params.value.textDocument);
-        const source_node = try document.nodeUnderCursor(params.value.position) orelse {
+        const source_node = try document.tokenUnderCursor(params.value.position) orelse {
             std.log.debug("no node under cursor", .{});
             return state.success(request.id, null);
         };

From 61f80f2f7b8dea6bfcedcb7e4e8fb5b4b2f75475 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 20:05:22 +0200
Subject: [PATCH 21/29] test: find-definition for fields

---
 src/analysis.zig | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/analysis.zig b/src/analysis.zig
index 7e5e171..4e7e439 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -735,6 +735,22 @@ test "find definition global" {
     });
 }
 
+test "find definition field" {
+    try expectDefinition(
+        \\struct Foo { int /*1*/bar, /*2*/baz; };
+        \\void main() {
+        \\    Foo foo;
+        \\    foo./*3*/bar;
+        \\    foo./*4*/baz;
+        \\}
+    , &.{
+        .{ .source = "/*3*/", .target = "/*1*/", .should_exist = true },
+        .{ .source = "/*3*/", .target = "/*2*/", .should_exist = false },
+        .{ .source = "/*4*/", .target = "/*1*/", .should_exist = false },
+        .{ .source = "/*4*/", .target = "/*2*/", .should_exist = true },
+    });
+}
+
 fn expectDefinition(
     source: []const u8,
     cases: []const struct {

From c932f8d3fc09f05fd2d308b58c27d9bf7c932e6a Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 20:25:48 +0200
Subject: [PATCH 22/29] fix off-by-one error in completions

---
 src/Document.zig | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Document.zig b/src/Document.zig
index a505233..efce057 100644
--- a/src/Document.zig
+++ b/src/Document.zig
@@ -113,7 +113,7 @@ pub fn tokenBeforeCursor(self: *@This(), cursor: lsp.Position) !?u32 {
             if (span.start == span.end) continue;
 
             // ignore tokens after the cursor
-            if (offset < span.start) continue;
+            if (offset <= span.start) continue;
 
             if (span.end > best_end) {
                 // found a token further to the right

From 63eb94f143510984694e57738469d5889ff27dec Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Tue, 17 Oct 2023 20:39:33 +0200
Subject: [PATCH 23/29] enable parallel ci builds for PRs

---
 .github/workflows/pull-request.yml | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index ea6d1dd..0edb97a 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -5,7 +5,16 @@ on:
 permissions:
   contents: write
 jobs:
-  build-artifacts:
+  build-pull-request:
+    strategy:
+      matrix:
+        target:
+          - "x86_64-linux-musl"
+          - "aarch64-linux-musl"
+          - "x86_64-macos"
+          - "aarch64-macos"
+          - "x86_64-windows"
+          - "aarch64-windows"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -14,9 +23,21 @@ jobs:
         with:
           version: 0.12.0-dev.790+ad6f8e3a5
       - name: Build
-        run: zig build release -Doptimize=Debug
+        run: zig build -Dtarget=${{ matrix.target }}
+  test-pull-request:
+    strategy:
+      matrix:
+        os:
+          - "ubuntu-latest"
+          - "windows-latest"
+          - "macos-latest"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Zig
+        uses: goto-bus-stop/setup-zig@v2.1.1
+        with:
+          version: 0.12.0-dev.790+ad6f8e3a5
       - name: Test
         run: zig build test
           
-
-

From 242ded2af2099d89a142cd646c87f961dc1c4b94 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Wed, 18 Oct 2023 19:52:46 +0200
Subject: [PATCH 24/29] disable recursive references in initializers

---
 src/analysis.zig | 178 +++++++++++++++++++++++++++++++++++------------
 src/parse.zig    |   2 +-
 src/syntax.zig   |   6 ++
 3 files changed, 141 insertions(+), 45 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index 4e7e439..c3a4951 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -435,15 +435,24 @@ fn collectLocalSymbols(
     defer path.deinit(arena);
 
     var closest_declaration: ?u32 = null;
+    var vardecls_before: u32 = std.math.maxInt(u32);
 
     {
         var child = target_node;
         while (tree.parent(child)) |parent| : (child = parent) {
-            if (closest_declaration == null and syntax.AnyDeclaration.match(tree, parent) != null) {
-                closest_declaration = parent;
-            }
-
             try path.append(arena, child);
+
+            if (closest_declaration == null) {
+                if (syntax.VariableDeclaration.tryExtract(tree, parent)) |vardecl| {
+                    var before = vardeclsBeforeInList(tree, vardecl) orelse 0;
+                    before += @intFromBool(child == vardecl.nodeOf(.name, tree));
+                    vardecls_before = before;
+                }
+
+                if (syntax.AnyDeclaration.match(tree, parent) != null) {
+                    closest_declaration = parent;
+                }
+            }
         }
     }
 
@@ -454,36 +463,67 @@ fn collectLocalSymbols(
 
         const children = tree.children(parent);
         var current_child = children.start;
-        while (current_child < target_child or current_child == closest_declaration) : (current_child += 1) {
-            if (syntax.ExternalDeclaration.tryExtract(tree, current_child)) |declaration| {
-                try collectDeclarationSymbols(scope, document, tree, declaration);
-                continue;
-            }
-            if (syntax.ParameterList.tryExtract(tree, current_child)) |parameters| {
-                var iterator = parameters.iterator();
-                while (iterator.next(tree)) |parameter| {
-                    const variable = parameter.get(.variable, tree) orelse continue;
-                    try registerVariables(scope, document, tree, .{ .one = variable }, parameter.node);
-                }
-                continue;
-            }
-            if (syntax.ConditionList.tryExtract(tree, current_child)) |condition_list| {
-                var statements = condition_list.iterator();
-                while (statements.next(tree)) |statement| {
-                    switch (statement) {
-                        .declaration => |declaration| {
-                            try collectDeclarationSymbols(
-                                scope,
-                                document,
-                                tree,
-                                .{ .variable = declaration },
-                            );
-                        },
-                    }
-                }
-                continue;
+        while (current_child < target_child) : (current_child += 1) {
+            try registerLocalDeclaration(scope, document, tree, current_child, .{});
+        }
+
+        if (current_child == closest_declaration) {
+            try registerLocalDeclaration(scope, document, tree, current_child, .{
+                .max_vardecl_count = vardecls_before,
+            });
+        }
+    }
+}
+
+fn vardeclsBeforeInList(tree: Tree, vardecl: syntax.VariableDeclaration) ?u32 {
+    const grandparent = tree.parent(vardecl.node) orelse return null;
+    if (tree.tag(grandparent) != .variable_declaration_list) return null;
+    const first = tree.children(grandparent).start;
+    return vardecl.node - first;
+}
+
+fn registerLocalDeclaration(
+    scope: *Scope,
+    document: *Document,
+    tree: Tree,
+    node: u32,
+    options: CollectOptions,
+) !void {
+    if (syntax.ExternalDeclaration.tryExtract(tree, node)) |declaration| {
+        try collectDeclarationSymbols(scope, document, tree, declaration, options);
+        return;
+    }
+
+    if (syntax.ParameterList.tryExtract(tree, node)) |parameters| {
+        var iterator = parameters.iterator();
+        while (iterator.next(tree)) |parameter| {
+            const variable = parameter.get(.variable, tree) orelse continue;
+            try registerVariables(scope, document, tree, .{ .one = variable }, parameter.node, .{});
+        }
+        return;
+    }
+
+    if (syntax.Parameter.tryExtract(tree, node)) |parameter| {
+        const variable = parameter.get(.variable, tree) orelse return;
+        try registerVariables(scope, document, tree, .{ .one = variable }, parameter.node, options);
+    }
+
+    if (syntax.ConditionList.tryExtract(tree, node)) |condition_list| {
+        var statements = condition_list.iterator();
+        while (statements.next(tree)) |statement| {
+            switch (statement) {
+                .declaration => |declaration| {
+                    try collectDeclarationSymbols(
+                        scope,
+                        document,
+                        tree,
+                        .{ .variable = declaration },
+                        .{},
+                    );
+                },
             }
         }
+        return;
     }
 }
 
@@ -494,15 +534,20 @@ fn collectGlobalSymbols(scope: *Scope, document: *Document) !void {
     const children = tree.children(tree.root);
     for (children.start..children.end) |child| {
         const global = syntax.ExternalDeclaration.tryExtract(tree, @intCast(child)) orelse continue;
-        try collectDeclarationSymbols(scope, document, tree, global);
+        try collectDeclarationSymbols(scope, document, tree, global, .{});
     }
 }
 
+const CollectOptions = struct {
+    max_vardecl_count: u32 = std.math.maxInt(u32),
+};
+
 fn collectDeclarationSymbols(
     scope: *Scope,
     document: *Document,
     tree: parse.Tree,
     external: syntax.ExternalDeclaration,
+    options: CollectOptions,
 ) !void {
     switch (external) {
         .function => |function| {
@@ -518,17 +563,17 @@ fn collectDeclarationSymbols(
             }
 
             const variables = declaration.get(.variables, tree) orelse return;
-            try registerVariables(scope, document, tree, variables, declaration.node);
+            try registerVariables(scope, document, tree, variables, declaration.node, options);
         },
         .block => |block| {
             if (block.get(.variable, tree)) |variable| {
-                try registerVariables(scope, document, tree, .{ .one = variable }, block.node);
+                try registerVariables(scope, document, tree, .{ .one = variable }, block.node, options);
             } else {
                 const fields = block.get(.fields, tree) orelse return;
                 var field_iterator = fields.iterator();
                 while (field_iterator.next(tree)) |field| {
                     const variables = field.get(.variables, tree) orelse continue;
-                    try registerVariables(scope, document, tree, variables, field.node);
+                    try registerVariables(scope, document, tree, variables, field.node, .{});
                 }
             }
         },
@@ -555,9 +600,14 @@ fn registerVariables(
     tree: Tree,
     variables: syntax.Variables,
     declaration: u32,
+    options: CollectOptions,
 ) !void {
+    var max_vardecl_count = options.max_vardecl_count;
     var iterator = variables.iterator();
     while (iterator.next(tree)) |variable| {
+        if (max_vardecl_count == 0) break;
+        max_vardecl_count -= 1;
+
         const name = variable.get(.name, tree) orelse return;
         const ident = name.getIdentifier(tree) orelse return;
         try registerIdentifier(scope, document, tree, ident, declaration);
@@ -751,6 +801,39 @@ test "find definition field" {
     });
 }
 
+test "find definition self" {
+    try expectDefinition(
+        \\void main(int /*1*/whatever) {
+        \\    float /*2*/foo;
+        \\}
+    , &.{
+        .{ .source = "/*1*/", .target = "/*1*/", .should_exist = true },
+        .{ .source = "/*2*/", .target = "/*2*/", .should_exist = true },
+    });
+}
+
+test "find definition self-multi" {
+    try expectDefinition(
+        \\void main() {
+        \\    float /*1*/foo = 123, bar = /*2*/foo;
+        \\}
+    , &.{
+        .{ .source = "/*2*/", .target = "/*1*/", .should_exist = true },
+    });
+}
+
+test "find definition local shadowing" {
+    try expectDefinition(
+        \\void main() {
+        \\    float /*1*/foo;
+        \\    int /*2*/foo = /*3*/foo;
+        \\}
+    , &.{
+        .{ .source = "/*3*/", .target = "/*1*/", .should_exist = true },
+        .{ .source = "/*3*/", .target = "/*2*/", .should_exist = false },
+    });
+}
+
 fn expectDefinition(
     source: []const u8,
     cases: []const struct {
@@ -771,6 +854,8 @@ fn expectDefinition(
     var cursors = try findCursors(document);
     defer cursors.deinit();
 
+    var print_source = false;
+
     for (cases) |case| {
         const usage = cursors.get(case.source) orelse std.debug.panic("invalid cursor: {s}", .{case.source});
         const definition = cursors.get(case.target) orelse std.debug.panic("invalid cursor: {s}", .{case.source});
@@ -788,19 +873,24 @@ fn expectDefinition(
         }
 
         if (case.should_exist and !found_definition) {
-            std.log.err(
-                "{s} did not find {s}:\n================\n{s}\n================",
-                .{ case.source, case.target, source },
-            );
+            std.log.err("definition not found: {s} -> {s}", .{ case.source, case.target });
+            print_source = true;
         }
 
         if (!case.should_exist and found_definition) {
-            std.log.err(
-                "{s} did find {s}:\n================\n{s}\n================",
-                .{ case.source, case.target, source },
-            );
+            std.log.err("unexpected definition: {s} -> {s}", .{ case.source, case.target });
+            print_source = true;
         }
     }
+
+    if (print_source) {
+        std.debug.print("================\n{s}\n================\n", .{source});
+        const parsed = try document.parseTree();
+        std.debug.print(
+            "================\n{}\n================\n",
+            .{parsed.tree.format(document.source())},
+        );
+    }
 }
 
 const Cursor = struct {
diff --git a/src/parse.zig b/src/parse.zig
index 619c7e0..67783ac 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -359,7 +359,7 @@ pub const Tree = struct {
         };
 
         var f = Formatter{ .tree = data.tree, .source = data.source, .writer = writer };
-        try f.writeNode(data.tree.nodes.len - 1);
+        try f.writeNode(data.tree.root);
     }
 };
 
diff --git a/src/syntax.zig b/src/syntax.zig
index 3cd3bfa..b64fe7c 100644
--- a/src/syntax.zig
+++ b/src/syntax.zig
@@ -276,6 +276,12 @@ pub fn Extractor(comptime expected_tag: Tag, comptime T: type) type {
             return .{ .node = node, .matches = matches };
         }
 
+        pub fn nodeOf(self: @This(), comptime field: FieldEnum, tree: Tree) ?u32 {
+            const field_match = @field(self.matches, @tagName(field));
+            const node_offset = field_match.node_offset orelse return null;
+            return tree.children(self.node).start + node_offset;
+        }
+
         pub fn get(self: @This(), comptime field: FieldEnum, tree: Tree) ?std.meta.FieldType(T, field) {
             const field_match = @field(self.matches, @tagName(field));
             const node_offset = field_match.node_offset orelse return null;

From 20a4d8e704fead1475d479c8a82b904d909271f1 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Wed, 18 Oct 2023 20:00:51 +0200
Subject: [PATCH 25/29] don't give completions in comments

---
 src/main.zig | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main.zig b/src/main.zig
index 56d7277..fd94009 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -547,6 +547,12 @@ pub const Dispatch = struct {
 
         const token = try document.tokenBeforeCursor(params.value.position);
 
+        const parsed = try document.parseTree();
+        if (token != null and parsed.tree.tag(token.?) == .comment) {
+            // don't give completions in comments
+            return state.success(request.id, null);
+        }
+
         try completionsAtToken(
             state,
             document,

From 5908c38116e24f445f778fe8f3e7ea994a87014f Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Wed, 18 Oct 2023 20:02:40 +0200
Subject: [PATCH 26/29] allow hovers on the character right after identifiers

---
 src/Document.zig | 4 ++--
 src/main.zig     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Document.zig b/src/Document.zig
index efce057..8a30dbd 100644
--- a/src/Document.zig
+++ b/src/Document.zig
@@ -84,13 +84,13 @@ pub fn nodeRange(self: *@This(), node: u32) !lsp.Range {
 }
 
 /// Return the node right under the cursor.
-pub fn tokenUnderCursor(self: *@This(), cursor: lsp.Position) !?u32 {
+pub fn identifierUnderCursor(self: *@This(), cursor: lsp.Position) !?u32 {
     const offset = self.utf8FromPosition(cursor);
     const parsed = try self.parseTree();
     const tree = parsed.tree;
 
     for (0.., tree.nodes.items(.tag), tree.nodes.items(.span)) |index, tag, span| {
-        if (tag.isToken() and span.start <= offset and offset < span.end) {
+        if (tag == .identifier and span.start <= offset and offset <= span.end) {
             return @intCast(index);
         }
     }
diff --git a/src/main.zig b/src/main.zig
index fd94009..db5bc42 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -645,7 +645,7 @@ pub const Dispatch = struct {
         const document = try getDocumentOrFail(state, request, params.value.textDocument);
         const parsed = try document.parseTree();
 
-        const token = try document.tokenUnderCursor(params.value.position) orelse {
+        const token = try document.identifierUnderCursor(params.value.position) orelse {
             return state.success(request.id, null);
         };
 
@@ -749,7 +749,7 @@ pub const Dispatch = struct {
         });
 
         const document = try state.workspace.getOrLoadDocument(params.value.textDocument);
-        const source_node = try document.tokenUnderCursor(params.value.position) orelse {
+        const source_node = try document.identifierUnderCursor(params.value.position) orelse {
             std.log.debug("no node under cursor", .{});
             return state.success(request.id, null);
         };

From 6bbd1ec4e5d413867240bd3282c1fab3a73ae676 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Wed, 18 Oct 2023 20:30:15 +0200
Subject: [PATCH 27/29] allow nested field completions

---
 src/analysis.zig | 54 ++++++++++++++++++++++++++++--------------------
 src/syntax.zig   |  7 +++++++
 2 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index c3a4951..e0098f0 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -227,35 +227,34 @@ pub fn visibleFields(
     start_node: u32,
     symbols: *std.ArrayList(Reference),
 ) !void {
-    const name = blk: {
+    const lhs = lhs: {
         const parsed = try document.parseTree();
-        const tag = parsed.tree.tag(start_node);
-        if (tag != .identifier and tag != .@".") return;
+        const tree = parsed.tree;
 
-        const parent = parsed.tree.parent(start_node) orelse return;
-        const parent_tag = parsed.tree.tag(parent);
-        if (parent_tag != .selection) return;
+        const tag = tree.tag(start_node);
+        if (tag != .identifier and tag != .@".") return;
 
-        const children = parsed.tree.children(parent);
-        if (start_node == children.start) return;
+        const parent = tree.parent(start_node) orelse return;
+        const selection = syntax.Selection.tryExtract(tree, parent) orelse return;
+        var target = selection.get(.target, tree) orelse return;
 
-        var first = children.start;
-        while (parsed.tree.tag(first).isSyntax()) {
-            const grand_children = parsed.tree.children(first);
-            if (grand_children.start == grand_children.end) return;
-            first = grand_children.start;
+        while (true) {
+            switch (target.get(tree)) {
+                .identifier => break :lhs target.node,
+                .selection => |select| break :lhs select.nodeOf(.field, tree) orelse return,
+                .array => |array| target = array.prefix(tree) orelse return,
+                .number => return,
+            }
         }
-        break :blk first;
     };
 
-    if (start_node == name) {
+    if (lhs == start_node) {
         // possible infinite loop if we are coming from `findDefinition`
         return;
     }
 
     var name_definitions = std.ArrayList(Reference).init(arena);
-    try findDefinition(arena, document, name, &name_definitions);
-    if (name_definitions.items.len == 0) return;
+    try findDefinition(arena, document, lhs, &name_definitions);
 
     var references = std.ArrayList(Reference).init(document.workspace.allocator);
     defer references.deinit();
@@ -801,6 +800,22 @@ test "find definition field" {
     });
 }
 
+test "find definition field recursive" {
+    try expectDefinition(
+        \\struct Foo { int /*1*/foo; };
+        \\struct Bar { Foo /*2*/bar; };
+        \\void main() {
+        \\    Bar baz;
+        \\    baz./*3*/bar./*4*/foo;
+        \\}
+    , &.{
+        .{ .source = "/*3*/", .target = "/*1*/", .should_exist = false },
+        .{ .source = "/*3*/", .target = "/*2*/", .should_exist = true },
+        .{ .source = "/*4*/", .target = "/*1*/", .should_exist = true },
+        .{ .source = "/*4*/", .target = "/*2*/", .should_exist = false },
+    });
+}
+
 test "find definition self" {
     try expectDefinition(
         \\void main(int /*1*/whatever) {
@@ -885,11 +900,6 @@ fn expectDefinition(
 
     if (print_source) {
         std.debug.print("================\n{s}\n================\n", .{source});
-        const parsed = try document.parseTree();
-        std.debug.print(
-            "================\n{}\n================\n",
-            .{parsed.tree.format(document.source())},
-        );
     }
 }
 
diff --git a/src/syntax.zig b/src/syntax.zig
index b64fe7c..feaee39 100644
--- a/src/syntax.zig
+++ b/src/syntax.zig
@@ -188,8 +188,15 @@ pub const ExpressionUnion = union(enum) {
     identifier: Token(.identifier),
     number: Token(.number),
     array: ArraySpecifier(Expression),
+    selection: Selection,
 };
 
+pub const Selection = Extractor(.selection, struct {
+    target: Expression,
+    @".": Token(.@"."),
+    field: Token(.identifier),
+});
+
 pub fn Token(comptime tag: Tag) type {
     comptime std.debug.assert(tag.isToken());
     return struct {

From 06f06d00cd965c6e8bcc1b83f00cbf9ab16ea851 Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Thu, 19 Oct 2023 21:26:25 +0200
Subject: [PATCH 28/29] only resolve last occurance of overloaded functions

---
 src/analysis.zig | 70 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 68 insertions(+), 2 deletions(-)

diff --git a/src/analysis.zig b/src/analysis.zig
index e0098f0..ddc1605 100644
--- a/src/analysis.zig
+++ b/src/analysis.zig
@@ -367,9 +367,25 @@ pub const Scope = struct {
         return std.mem.lastIndexOfScalar(ScopeId, self.active_scopes.items, scope) != null;
     }
 
-    pub fn getVisible(self: *const @This(), symbols: *std.ArrayList(Reference)) !void {
+    pub fn getVisible(
+        self: *const @This(),
+        symbols: *std.ArrayList(Reference),
+        options: struct {
+            /// An allocator used to detect duplicates.
+            duplicate_allocator: std.mem.Allocator,
+        },
+    ) !void {
+        var functions = std.StringHashMap(void).init(options.duplicate_allocator);
+        defer functions.deinit();
+
+        var arena = std.heap.ArenaAllocator.init(options.duplicate_allocator);
+        defer arena.deinit();
+
         try symbols.ensureUnusedCapacity(self.symbols.count());
         for (self.symbols.values()) |*value| {
+            defer functions.clearRetainingCapacity();
+            defer _ = arena.reset(.retain_capacity);
+
             var first_scope: ?u32 = null;
             var current: ?*Symbol = value;
             while (current) |symbol| : (current = symbol.shadowed) {
@@ -379,6 +395,14 @@ pub const Scope = struct {
                 first_scope = first_scope orelse symbol.scope;
                 if (symbol.scope != first_scope) break;
 
+                if (try functionParameterSignature(
+                    arena.allocator(),
+                    symbol.reference,
+                )) |signature| {
+                    const result = try functions.getOrPut(signature);
+                    if (result.found_existing) continue;
+                }
+
                 try symbols.append(symbol.reference);
 
                 // only symbols in global scope can be overloaded
@@ -386,6 +410,33 @@ pub const Scope = struct {
             }
         }
     }
+
+    fn functionParameterSignature(allocator: std.mem.Allocator, reference: Reference) !?[]u8 {
+        const document = reference.document;
+        const parsed = try document.parseTree();
+        const tree = parsed.tree;
+        const decl_node = reference.parent_declaration;
+
+        const func = syntax.FunctionDeclaration.tryExtract(tree, decl_node) orelse return null;
+        const parameters = func.get(.parameters, tree) orelse return null;
+
+        var signature = std.ArrayList(u8).init(allocator);
+        errdefer signature.deinit();
+
+        try signature.appendSlice("(");
+
+        var i: usize = 0;
+        var iterator = parameters.iterator();
+        while (iterator.next(tree)) |parameter| : (i += 1) {
+            if (i != 0) try signature.appendSlice(", ");
+            const typ = parameterType(parameter, tree);
+            try signature.writer().print("{}", .{typ.format(tree, document.source())});
+        }
+
+        try signature.appendSlice(")");
+
+        return try signature.toOwnedSlice();
+    }
 };
 
 /// Get a list of all symbols visible starting from the given syntax node.
@@ -417,7 +468,7 @@ pub fn visibleSymbols(
         try scope.begin();
         defer scope.end();
         try collectLocalSymbols(arena, &scope, start_document, start_node);
-        try scope.getVisible(symbols);
+        try scope.getVisible(symbols, .{ .duplicate_allocator = arena });
     }
 }
 
@@ -849,6 +900,21 @@ test "find definition local shadowing" {
     });
 }
 
+test "find definition duplicate overload" {
+    try expectDefinition(
+        \\float /*1*/add(float a, float b);
+        \\int /*2*/add(int a, int b);
+        \\void main() {
+        \\    /*3*/add(1, 2);
+        \\}
+        \\int /*4*/add(int a, int b) { return a + b; }
+    , &.{
+        .{ .source = "/*3*/", .target = "/*1*/", .should_exist = true },
+        .{ .source = "/*3*/", .target = "/*2*/", .should_exist = false },
+        .{ .source = "/*2*/", .target = "/*4*/", .should_exist = true },
+    });
+}
+
 fn expectDefinition(
     source: []const u8,
     cases: []const struct {

From 8f651a4b0048dfed31425d4ece0728491d1ad63a Mon Sep 17 00:00:00 2001
From: Christofer Nolander <christofer@nolander.me>
Date: Thu, 19 Oct 2023 22:10:45 +0200
Subject: [PATCH 29/29] show extension information in hover documentation

---
 src/Document.zig  | 33 +++++++++++++++++++++++----------
 src/Workspace.zig | 42 ++++++++++++++++++++++++++----------------
 src/parse.zig     | 19 +++++++++++++++++++
 3 files changed, 68 insertions(+), 26 deletions(-)

diff --git a/src/Document.zig b/src/Document.zig
index 8a30dbd..a8c4418 100644
--- a/src/Document.zig
+++ b/src/Document.zig
@@ -126,10 +126,6 @@ pub fn tokenBeforeCursor(self: *@This(), cursor: lsp.Position) !?u32 {
     return best;
 }
 
-fn isIdentifierChar(c: u8) bool {
-    return std.ascii.isAlphanumeric(c) or c == '_';
-}
-
 pub fn parseTree(self: *@This()) !*const CompleteParseTree {
     if (self.parse_tree) |*tree| return tree;
     self.parse_tree = try CompleteParseTree.parseSource(
@@ -142,8 +138,11 @@ pub fn parseTree(self: *@This()) !*const CompleteParseTree {
 pub const CompleteParseTree = struct {
     arena_state: std.heap.ArenaAllocator.State,
     tree: parse.Tree,
-    ignored: []parse.Token,
-    diagnostics: std.ArrayListUnmanaged(parse.Diagnostic),
+    ignored: []const parse.Token,
+    diagnostics: []const parse.Diagnostic,
+
+    // List of enabled extensions
+    extensions: []const []const u8,
 
     pub fn deinit(self: *@This(), allocator: std.mem.Allocator) void {
         self.arena_state.promote(allocator).deinit();
@@ -153,22 +152,36 @@ pub const CompleteParseTree = struct {
         var arena = std.heap.ArenaAllocator.init(parent_allocator);
         errdefer arena.deinit();
 
-        const allocator = arena.allocator();
+        var diagnostics = std.ArrayList(parse.Diagnostic).init(arena.allocator());
 
-        var diagnostics = std.ArrayList(parse.Diagnostic).init(allocator);
         var ignored = std.ArrayList(parse.Token).init(parent_allocator);
         defer ignored.deinit();
 
-        const tree = try parse.parse(allocator, text, .{
+        const tree = try parse.parse(arena.allocator(), text, .{
             .ignored = &ignored,
             .diagnostics = &diagnostics,
         });
 
+        var extensions = std.ArrayList([]const u8).init(arena.allocator());
+        errdefer extensions.deinit();
+
+        for (ignored.items) |token| {
+            const line = text[token.start..token.end];
+            switch (parse.parsePreprocessorDirective(line) orelse continue) {
+                .extension => |extension| {
+                    const name = extension.name;
+                    try extensions.append(line[name.start..name.end]);
+                },
+                else => continue,
+            }
+        }
+
         return .{
             .arena_state = arena.state,
             .tree = tree,
             .ignored = tree.ignored(),
-            .diagnostics = diagnostics.moveToUnmanaged(),
+            .diagnostics = diagnostics.items,
+            .extensions = extensions.items,
         };
     }
 };
diff --git a/src/Workspace.zig b/src/Workspace.zig
index 2c7dd48..4db7ffd 100644
--- a/src/Workspace.zig
+++ b/src/Workspace.zig
@@ -163,40 +163,50 @@ fn builtinCompletions(arena: std.mem.Allocator, spec: *const Spec) ![]lsp.Comple
 
         try completions.append(.{
             .label = variable.name,
-            .labelDetails = .{
-                .detail = try signature.toOwnedSlice(),
-            },
+            .labelDetails = .{ .detail = signature.items },
             .kind = .variable,
-            .documentation = if (variable.description) |desc| .{
-                .kind = .markdown,
-                .value = try std.mem.join(arena, "\n\n", desc),
-            } else null,
+            .documentation = try itemDocumentation(arena, variable),
         });
     }
 
     for (spec.functions) |function| {
         var anonymous_signature = std.ArrayList(u8).init(arena);
-        var named_signature = std.ArrayList(u8).init(arena);
         try writeFunctionSignature(function, anonymous_signature.writer(), .{ .names = false });
+
+        var named_signature = std.ArrayList(u8).init(arena);
         try writeFunctionSignature(function, named_signature.writer(), .{ .names = true });
 
         try completions.append(.{
             .label = function.name,
-            .labelDetails = .{
-                .detail = try anonymous_signature.toOwnedSlice(),
-            },
+            .labelDetails = .{ .detail = anonymous_signature.items },
             .kind = .function,
-            .detail = try named_signature.toOwnedSlice(),
-            .documentation = if (function.description) |desc| .{
-                .kind = .markdown,
-                .value = try std.mem.join(arena, "\n\n", desc),
-            } else null,
+            .detail = named_signature.items,
+            .documentation = try itemDocumentation(arena, function),
         });
     }
 
     return completions.toOwnedSlice();
 }
 
+fn itemDocumentation(arena: std.mem.Allocator, item: anytype) !lsp.MarkupContent {
+    var documentation = std.ArrayList(u8).init(arena);
+
+    for (item.description orelse &.{}) |paragraph| {
+        try documentation.appendSlice(paragraph);
+        try documentation.appendSlice("\n\n");
+    }
+
+    if (item.extensions) |extensions| {
+        try documentation.appendSlice("```glsl\n");
+        for (extensions) |extension| {
+            try documentation.writer().print("#extension {s} : enable\n", .{extension});
+        }
+        try documentation.appendSlice("```\n");
+    }
+
+    return .{ .kind = .markdown, .value = try documentation.toOwnedSlice() };
+}
+
 fn writeFunctionSignature(
     function: Spec.Function,
     writer: anytype,
diff --git a/src/parse.zig b/src/parse.zig
index 67783ac..7321210 100644
--- a/src/parse.zig
+++ b/src/parse.zig
@@ -424,6 +424,9 @@ pub const Parser = struct {
 
         tree.root = @intCast(tree.nodes.len - 1);
 
+        // Append any ignored tokens to the end of the parse tree.
+        // This ensures that we can refer to ignored tokens with the same
+        // mechanisms as any other syntax node.
         if (self.options.ignored) |ignored| {
             const ignored_start = tree.nodes.len;
             try tree.nodes.resize(self.allocator, tree.nodes.len + ignored.items.len);
@@ -1630,6 +1633,8 @@ fn stripPrefix(text: []const u8, prefix: []const u8) ?[]const u8 {
 pub const Directive = union(enum) {
     define: struct { name: Span },
     include: struct { path: Span },
+    version: struct { number: Span },
+    extension: struct { name: Span },
 };
 
 pub fn parsePreprocessorDirective(line: []const u8) ?Directive {
@@ -1670,6 +1675,20 @@ pub fn parsePreprocessorDirective(line: []const u8) ?Directive {
         return .{ .include = .{ .path = .{ .start = path_start, .end = path_end } } };
     }
 
+    if (std.mem.eql(u8, kind, "extension")) {
+        const name_start = i;
+        const name_end = skipIdentifier(i, line);
+        if (name_start == name_end) return null;
+        return .{ .extension = .{ .name = .{ .start = name_start, .end = name_end } } };
+    }
+
+    if (std.mem.eql(u8, kind, "version")) {
+        const number_start = i;
+        const number_end = skipIdentifier(i, line);
+        if (number_start == number_end) return null;
+        return .{ .version = .{ .number = .{ .start = number_start, .end = number_end } } };
+    }
+
     return null;
 }