Update lexing and parsing

assets/tokens.yaml

@@ -394,7 +394,7 @@
   variant: Whitespace
   doc: whitespace.
 # after this point comes high-level tokens used in grammar
-- {kind: "syntax", variant: "Root", doc: "the root node of a parse tree"}
+- {kind: "syntax", variant: "Root", doc: "a syntax root"}
 - {kind: "syntax", variant: "Local", doc: "a variable declaration"}
 - {kind: "syntax", variant: "Item", doc: "an item declaration"}
 - {kind: "syntax", variant: "ItemEnum", doc: "an enum declaration"}
@@ -464,6 +464,7 @@
 - {kind: "syntax", variant: "ClosureArguments", doc: "closure arguments"}
 - {kind: "syntax", variant: "AnonymousObjectKey", doc: "an `#{` anonymous object key"}
 - {kind: "syntax", variant: "Attribute", doc: "an attribute"}
+- {kind: "syntax", variant: "InnerAttribute", doc: "an inner attribute"}
 - {kind: "syntax", variant: "Modifiers", doc: "item modifiers, like `pub const`"}
 - {kind: "syntax", variant: "ModifierCrate", doc: "the `(crate)` modifier"}
 - {kind: "syntax", variant: "ModifierIn", doc: "the `(in <path>)` modifier"}

crates/rune/Cargo.toml

@@ -48,7 +48,7 @@ serde = { version = "1.0.163", default-features = false, features = ["derive", "
 musli = { version = "0.0.121", default-features = false, features = ["alloc"] }
 once_cell = { version = "1.18.0", default-features = false, features = ["critical-section"] }
 
-syntree = { version = "0.17.0", optional = true }
+syntree = { version = "0.17.2", optional = true }
 anyhow = { version = "1.0.71", default-features = false, optional = true }
 bincode = { version = "1.3.3", optional = true }
 clap = { version = "4.2.7", features = ["derive"], optional = true }

crates/rune/src/compile/options.rs

@@ -112,6 +112,8 @@ pub struct Options {
     pub(crate) test_std: bool,
     /// Enable lowering optimizations.
     pub(crate) lowering: u8,
+    /// Print source tree.
+    pub(crate) print_tree: bool,
     /// Rune format options.
     pub(crate) fmt: FmtOptions,
 }
@@ -127,6 +129,7 @@ impl Options {
         function_body: false,
         test_std: false,
         lowering: 0,
+        print_tree: false,
         fmt: FmtOptions::DEFAULT,
     };
 
@@ -220,6 +223,16 @@ impl Options {
                 default: "0",
                 options: "0-3",
             },
+            OptionMeta {
+                key: "print-tree",
+                unstable: false,
+                doc: &docstring! {
+                    /// Print the parsed source tree when formatting to
+                    /// standard output.
+                },
+                default: "false",
+                options: BOOL,
+            },
             OptionMeta {
                 key: "fmt.print-tree",
                 unstable: false,
@@ -302,6 +315,9 @@ impl Options {
                     }
                 };
             }
+            "print-tree" => {
+                self.print_tree = tail.map_or(true, |s| s == "true");
+            }
             other => {
                 let Some((head, tail)) = other.split_once('.') else {
                     return Err(ParseOptionError {

crates/rune/src/fmt/format.rs

@@ -63,6 +63,15 @@ fn expr_labels<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> {
     Ok(())
 }
 
+fn inner_attributes<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> {
+    while let Some(attr) = p.try_pump(InnerAttribute)? {
+        attr.fmt(fmt)?;
+        fmt.nl(1)?;
+    }
+
+    Ok(())
+}
+
 fn attributes<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<Attrs> {
     let mut attrs = Attrs::default();
 
@@ -1611,6 +1620,8 @@ enum StmtKind {
 
 /// The contents of a block.
 fn block_content<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> {
+    inner_attributes(fmt, p)?;
+
     let mut last_kind = StmtKind::None;
 
     while !p.is_eof() {

crates/rune/src/grammar/grammar.rs

@@ -120,6 +120,8 @@ fn stmt(p: &mut Parser<'_>) -> Result<()> {
         }
     }
 
+    inner_attributes(p)?;
+
     let c = p.checkpoint()?;
     attributes(p)?;
     let m = modifiers(p)?;
@@ -187,8 +189,8 @@ fn item_const(p: &mut Parser<'_>) -> Result<()> {
 }
 
 #[tracing::instrument(skip_all)]
-fn attributes(p: &mut Parser<'_>) -> Result<()> {
-    while matches!((p.peek()?, p.glued(1)?), (K![#], K![!]) | (K![#], K!['['])) {
+fn inner_attributes(p: &mut Parser<'_>) -> Result<()> {
+    while matches!((p.peek()?, p.glued(1)?), (K![#], K![!])) {
         let c = p.checkpoint()?;
 
         p.bump()?;
@@ -199,6 +201,24 @@ fn attributes(p: &mut Parser<'_>) -> Result<()> {
             p.bump()?;
         }
 
+        p.close_at(&c, InnerAttribute)?;
+    }
+
+    Ok(())
+}
+
+#[tracing::instrument(skip_all)]
+fn attributes(p: &mut Parser<'_>) -> Result<()> {
+    while matches!((p.peek()?, p.glued(1)?), (K![#], K!['['])) {
+        let c = p.checkpoint()?;
+
+        p.bump()?;
+
+        if p.bump_if(K!['['])? {
+            token_stream(p, brackets)?;
+            p.bump()?;
+        }
+
         p.close_at(&c, Attribute)?;
     }
 

crates/rune/src/grammar/parser.rs

@@ -95,9 +95,9 @@ impl<'a> Parser<'a> {
     pub(super) fn bump(&mut self) -> Result<Token> {
         let tok = self.next()?;
 
-        self.tree
-            .token(tok.kind, tok.span.range().len())
-            .with_span(tok.span)?;
+        let span = syntree::Span::new(tok.span.start.0, tok.span.end.0);
+
+        self.tree.token_with(tok.kind, span).with_span(tok.span)?;
 
         Ok(tok)
     }
@@ -143,9 +143,11 @@ impl<'a> Parser<'a> {
     pub(super) fn push(&mut self, kind: Kind) -> Result<()> {
         let tok = self.next()?;
         self.tree.open(kind).with_span(tok.span)?;
-        self.tree
-            .token(tok.kind, tok.span.range().len())
-            .with_span(tok.span)?;
+
+        let span = syntree::Span::new(tok.span.start.0, tok.span.end.0);
+
+        self.tree.token_with(tok.kind, span).with_span(tok.span)?;
+
         self.tree.close().with_span(tok.span)?;
         Ok(())
     }

crates/rune/src/tests.rs

@@ -27,7 +27,6 @@ pub(crate) mod prelude {
     };
     pub(crate) use futures_executor::block_on;
 
-    pub(crate) use ::rust_alloc::borrow::ToOwned;
     pub(crate) use ::rust_alloc::boxed::Box;
     pub(crate) use ::rust_alloc::string::{String, ToString};
     pub(crate) use ::rust_alloc::sync::Arc;