From 67446ee9d003fec4df7792e4c4a2fb1febf5fdac Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Wed, 14 Aug 2024 13:56:40 +0200 Subject: [PATCH] Update lexing and parsing --- assets/tokens.yaml | 3 ++- crates/rune/Cargo.toml | 2 +- crates/rune/src/ast/generated.rs | 10 +++++----- crates/rune/src/compile/options.rs | 16 ++++++++++++++++ crates/rune/src/fmt/format.rs | 11 +++++++++++ crates/rune/src/grammar/grammar.rs | 24 ++++++++++++++++++++++-- crates/rune/src/grammar/parser.rs | 14 ++++++++------ crates/rune/src/tests.rs | 1 - 8 files changed, 65 insertions(+), 16 deletions(-) diff --git a/assets/tokens.yaml b/assets/tokens.yaml index 93310417f..8288e584b 100644 --- a/assets/tokens.yaml +++ b/assets/tokens.yaml @@ -394,7 +394,7 @@ variant: Whitespace doc: whitespace. # after this point comes high-level tokens used in grammar -- {kind: "syntax", variant: "Root", doc: "the root node of a parse tree"} +- {kind: "syntax", variant: "Root", doc: "a syntax root"} - {kind: "syntax", variant: "Local", doc: "a variable declaration"} - {kind: "syntax", variant: "Item", doc: "an item declaration"} - {kind: "syntax", variant: "ItemEnum", doc: "an enum declaration"} @@ -464,6 +464,7 @@ - {kind: "syntax", variant: "ClosureArguments", doc: "closure arguments"} - {kind: "syntax", variant: "AnonymousObjectKey", doc: "an `#{` anonymous object key"} - {kind: "syntax", variant: "Attribute", doc: "an attribute"} +- {kind: "syntax", variant: "InnerAttribute", doc: "an inner attribute"} - {kind: "syntax", variant: "Modifiers", doc: "item modifiers, like `pub const`"} - {kind: "syntax", variant: "ModifierCrate", doc: "the `(crate)` modifier"} - {kind: "syntax", variant: "ModifierIn", doc: "the `(in )` modifier"} diff --git a/crates/rune/Cargo.toml b/crates/rune/Cargo.toml index bccf172d2..4978f1c67 100644 --- a/crates/rune/Cargo.toml +++ b/crates/rune/Cargo.toml @@ -48,7 +48,7 @@ serde = { version = "1.0.163", default-features = false, features = ["derive", " musli = { version = "0.0.121", default-features = false, features = ["alloc"] } once_cell = { version = "1.18.0", default-features = false, features = ["critical-section"] } -syntree = { version = "0.17.0", optional = true } +syntree = { version = "0.17.2", optional = true } anyhow = { version = "1.0.71", default-features = false, optional = true } bincode = { version = "1.3.3", optional = true } clap = { version = "4.2.7", features = ["derive"], optional = true } diff --git a/crates/rune/src/ast/generated.rs b/crates/rune/src/ast/generated.rs index df63c13a4..8894b6a62 100644 --- a/crates/rune/src/ast/generated.rs +++ b/crates/rune/src/ast/generated.rs @@ -5680,7 +5680,7 @@ pub enum Kind { Yield, /// whitespace. Whitespace, - /// the root node of a parse tree + /// a syntax root Root, /// a variable declaration Local, @@ -5704,8 +5704,6 @@ pub enum Kind { ItemUsePath, /// a nested use group ItemUseGroup, - /// the variants part of an enum - EnumVariants, /// a variant Variant, /// a field declaration @@ -5822,6 +5820,8 @@ pub enum Kind { AnonymousObjectKey, /// an attribute Attribute, + /// an inner attribute + InnerAttribute, /// item modifiers, like `pub const` Modifiers, /// the `(crate)` modifier @@ -6143,7 +6143,7 @@ impl parse::IntoExpectation for Kind { Self::Tilde => parse::Expectation::Punctuation("~"), Self::Underscore => parse::Expectation::Punctuation("_"), Self::Whitespace => parse::Expectation::Syntax("whitespace."), - Self::Root => parse::Expectation::Syntax("the root node of a parse tree"), + Self::Root => parse::Expectation::Syntax("a syntax root"), Self::Local => parse::Expectation::Syntax("a variable declaration"), Self::Item => parse::Expectation::Syntax("an item declaration"), Self::ItemEnum => parse::Expectation::Syntax("an enum declaration"), @@ -6155,7 +6155,6 @@ impl parse::IntoExpectation for Kind { Self::ItemUse => parse::Expectation::Syntax("a use import"), Self::ItemUsePath => parse::Expectation::Syntax("a nested use path"), Self::ItemUseGroup => parse::Expectation::Syntax("a nested use group"), - Self::EnumVariants => parse::Expectation::Syntax("the variants part of an enum"), Self::Variant => parse::Expectation::Syntax("a variant"), Self::Field => parse::Expectation::Syntax("a field declaration"), Self::EmptyBody => parse::Expectation::Syntax("an empty type body"), @@ -6218,6 +6217,7 @@ impl parse::IntoExpectation for Kind { Self::ClosureArguments => parse::Expectation::Syntax("closure arguments"), Self::AnonymousObjectKey => parse::Expectation::Syntax("an `#{` anonymous object key"), Self::Attribute => parse::Expectation::Syntax("an attribute"), + Self::InnerAttribute => parse::Expectation::Syntax("an inner attribute"), Self::Modifiers => parse::Expectation::Syntax("item modifiers, like `pub const`"), Self::ModifierCrate => parse::Expectation::Syntax("the `(crate)` modifier"), Self::ModifierIn => parse::Expectation::Syntax("the `(in )` modifier"), diff --git a/crates/rune/src/compile/options.rs b/crates/rune/src/compile/options.rs index c4e1d6837..29877eec7 100644 --- a/crates/rune/src/compile/options.rs +++ b/crates/rune/src/compile/options.rs @@ -112,6 +112,8 @@ pub struct Options { pub(crate) test_std: bool, /// Enable lowering optimizations. pub(crate) lowering: u8, + /// Print source tree. + pub(crate) print_tree: bool, /// Rune format options. pub(crate) fmt: FmtOptions, } @@ -127,6 +129,7 @@ impl Options { function_body: false, test_std: false, lowering: 0, + print_tree: false, fmt: FmtOptions::DEFAULT, }; @@ -220,6 +223,16 @@ impl Options { default: "0", options: "0-3", }, + OptionMeta { + key: "print-tree", + unstable: false, + doc: &docstring! { + /// Print the parsed source tree when formatting to + /// standard output. + }, + default: "false", + options: BOOL, + }, OptionMeta { key: "fmt.print-tree", unstable: false, @@ -302,6 +315,9 @@ impl Options { } }; } + "print-tree" => { + self.print_tree = tail.map_or(true, |s| s == "true"); + } other => { let Some((head, tail)) = other.split_once('.') else { return Err(ParseOptionError { diff --git a/crates/rune/src/fmt/format.rs b/crates/rune/src/fmt/format.rs index e6ba2d31e..c3b1b5e0b 100644 --- a/crates/rune/src/fmt/format.rs +++ b/crates/rune/src/fmt/format.rs @@ -63,6 +63,15 @@ fn expr_labels<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> { Ok(()) } +fn inner_attributes<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> { + while let Some(attr) = p.try_pump(InnerAttribute)? { + attr.fmt(fmt)?; + fmt.nl(1)?; + } + + Ok(()) +} + fn attributes<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result { let mut attrs = Attrs::default(); @@ -1611,6 +1620,8 @@ enum StmtKind { /// The contents of a block. fn block_content<'a>(fmt: &mut Formatter<'a>, p: &mut Stream<'a>) -> Result<()> { + inner_attributes(fmt, p)?; + let mut last_kind = StmtKind::None; while !p.is_eof() { diff --git a/crates/rune/src/grammar/grammar.rs b/crates/rune/src/grammar/grammar.rs index 33f4df1f3..70e1d1f9b 100644 --- a/crates/rune/src/grammar/grammar.rs +++ b/crates/rune/src/grammar/grammar.rs @@ -120,6 +120,8 @@ fn stmt(p: &mut Parser<'_>) -> Result<()> { } } + inner_attributes(p)?; + let c = p.checkpoint()?; attributes(p)?; let m = modifiers(p)?; @@ -187,8 +189,8 @@ fn item_const(p: &mut Parser<'_>) -> Result<()> { } #[tracing::instrument(skip_all)] -fn attributes(p: &mut Parser<'_>) -> Result<()> { - while matches!((p.peek()?, p.glued(1)?), (K![#], K![!]) | (K![#], K!['['])) { +fn inner_attributes(p: &mut Parser<'_>) -> Result<()> { + while matches!((p.peek()?, p.glued(1)?), (K![#], K![!])) { let c = p.checkpoint()?; p.bump()?; @@ -199,6 +201,24 @@ fn attributes(p: &mut Parser<'_>) -> Result<()> { p.bump()?; } + p.close_at(&c, InnerAttribute)?; + } + + Ok(()) +} + +#[tracing::instrument(skip_all)] +fn attributes(p: &mut Parser<'_>) -> Result<()> { + while matches!((p.peek()?, p.glued(1)?), (K![#], K!['['])) { + let c = p.checkpoint()?; + + p.bump()?; + + if p.bump_if(K!['['])? { + token_stream(p, brackets)?; + p.bump()?; + } + p.close_at(&c, Attribute)?; } diff --git a/crates/rune/src/grammar/parser.rs b/crates/rune/src/grammar/parser.rs index 689cc5384..2c5d5f691 100644 --- a/crates/rune/src/grammar/parser.rs +++ b/crates/rune/src/grammar/parser.rs @@ -95,9 +95,9 @@ impl<'a> Parser<'a> { pub(super) fn bump(&mut self) -> Result { let tok = self.next()?; - self.tree - .token(tok.kind, tok.span.range().len()) - .with_span(tok.span)?; + let span = syntree::Span::new(tok.span.start.0, tok.span.end.0); + + self.tree.token_with(tok.kind, span).with_span(tok.span)?; Ok(tok) } @@ -143,9 +143,11 @@ impl<'a> Parser<'a> { pub(super) fn push(&mut self, kind: Kind) -> Result<()> { let tok = self.next()?; self.tree.open(kind).with_span(tok.span)?; - self.tree - .token(tok.kind, tok.span.range().len()) - .with_span(tok.span)?; + + let span = syntree::Span::new(tok.span.start.0, tok.span.end.0); + + self.tree.token_with(tok.kind, span).with_span(tok.span)?; + self.tree.close().with_span(tok.span)?; Ok(()) } diff --git a/crates/rune/src/tests.rs b/crates/rune/src/tests.rs index 653928ad0..020afb78e 100644 --- a/crates/rune/src/tests.rs +++ b/crates/rune/src/tests.rs @@ -27,7 +27,6 @@ pub(crate) mod prelude { }; pub(crate) use futures_executor::block_on; - pub(crate) use ::rust_alloc::borrow::ToOwned; pub(crate) use ::rust_alloc::boxed::Box; pub(crate) use ::rust_alloc::string::{String, ToString}; pub(crate) use ::rust_alloc::sync::Arc;