From 12cbe054c57a6d21775c371edaf0b21d9dda6c5c Mon Sep 17 00:00:00 2001 From: Olivier FAURE Date: Mon, 11 Apr 2022 18:16:59 +0200 Subject: [PATCH] Add basic expression parsing --- src/lib.rs | 6 +- src/parse.rs | 103 +++++++++++-- ...venial__tests__parse_basic_expression.snap | 14 ++ ...l__tests__parse_empty_expression_list.snap | 6 + ...enial__tests__parse_enum_discriminant.snap | 143 ++++++++++-------- ...al__tests__parse_expression_binary_or.snap | 17 +++ ...l__tests__parse_expression_list_items.snap | 15 ++ ...ests__parse_expression_list_turbofish.snap | 24 +++ src/tests.rs | 62 +++++++- src/types.rs | 39 +++-- 10 files changed, 335 insertions(+), 94 deletions(-) create mode 100644 src/snapshots/venial__tests__parse_basic_expression.snap create mode 100644 src/snapshots/venial__tests__parse_empty_expression_list.snap create mode 100644 src/snapshots/venial__tests__parse_expression_binary_or.snap create mode 100644 src/snapshots/venial__tests__parse_expression_list_items.snap create mode 100644 src/snapshots/venial__tests__parse_expression_list_turbofish.snap diff --git a/src/lib.rs b/src/lib.rs index 065cd6d..49e8a10 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,10 +68,10 @@ mod types; mod types_edition; pub use error::Error; -pub use parse::parse_declaration; +pub use parse::{parse_declaration, parse_expression_list}; pub use punctuated::Punctuated; pub use types::{ - Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, GenericBound, GenericParam, - GenericParams, NamedField, NamedStructFields, Struct, StructFields, TupleField, + Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, Expression, GenericBound, + GenericParam, GenericParams, NamedField, NamedStructFields, Struct, StructFields, TupleField, TupleStructFields, TyExpr, Union, VisMarker, WhereClause, WhereClauseItem, }; diff --git a/src/parse.rs b/src/parse.rs index bc0d467..aebbf15 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,11 +1,11 @@ use crate::punctuated::Punctuated; use crate::types::{ - Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, Function, FunctionParameter, - FunctionQualifiers, GenericBound, GenericParam, GenericParams, NamedField, NamedStructFields, - Struct, StructFields, TupleField, TupleStructFields, TyExpr, Union, VisMarker, WhereClause, - WhereClauseItem, + Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, Expression, Function, + FunctionParameter, FunctionQualifiers, GenericBound, GenericParam, GenericParams, NamedField, + NamedStructFields, Struct, StructFields, TupleField, TupleStructFields, TyExpr, Union, + VisMarker, WhereClause, WhereClauseItem, }; -use proc_macro2::{Delimiter, Group, Ident, Punct, TokenStream, TokenTree}; +use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, TokenStream, TokenTree}; use std::iter::Peekable; type TokenIter = Peekable; @@ -85,7 +85,7 @@ fn consume_declaration_name(tokens: &mut TokenIter) -> Ident { // Consumes tokens until a separator is reached *unless* the // separator in between angle brackets -// eg consume_stuff_until(..., ',') will consume all +// eg consume_stuff_until(..., |token| token == ',') will consume all // of `Foobar,` except for the last comma pub(crate) fn consume_stuff_until( tokens: &mut TokenIter, @@ -96,8 +96,6 @@ pub(crate) fn consume_stuff_until( let mut predicate = predicate; let mut prev_token_is_dash = false; - // TODO - handle closures - loop { let token = tokens.peek(); prev_token_is_dash = match &token { @@ -131,6 +129,64 @@ pub(crate) fn consume_stuff_until( output_tokens } +// Consumes tokens until a comma is reached, except in +// various corner cases related to expression syntax. +// eg consume_expression(...) will consume all +// of `a + |b, c| d, e::(), h,` except for the last comma +pub(crate) fn consume_expression(tokens: &mut TokenIter) -> Expression { + let mut output_tokens = Vec::new(); + + // TODO - handle closures + // TODO - handle ::xxx syntax + + // TODO - use matches instead? + #[derive(Debug, PartialEq)] + enum PrevToken { + Any, + FirstColon, + DoubleColon, + } + let mut prev_token = PrevToken::Any; + + loop { + let token = tokens.peek(); + prev_token = match &token { + Some(TokenTree::Punct(punct)) + if punct.as_char() == ':' && punct.spacing() == Spacing::Joint => + { + output_tokens.push(tokens.next().unwrap()); + PrevToken::FirstColon + } + Some(TokenTree::Punct(punct)) + if punct.as_char() == ':' && prev_token == PrevToken::FirstColon => + { + output_tokens.push(tokens.next().unwrap()); + PrevToken::DoubleColon + } + + Some(TokenTree::Punct(punct)) + if punct.as_char() == '<' && prev_token == PrevToken::DoubleColon => + { + let mut turbofish_contents = consume_stuff_until(tokens, |_| true); + output_tokens.append(&mut turbofish_contents); + PrevToken::Any + } + + Some(TokenTree::Punct(punct)) if punct.as_char() == ',' => break, + None => break, + + _ => { + output_tokens.push(tokens.next().unwrap()); + PrevToken::Any + } + }; + } + + Expression { + tokens: output_tokens, + } +} + fn consume_comma(tokens: &mut TokenIter) -> Option { match tokens.peek() { Some(TokenTree::Punct(punct)) if punct.as_char() == ',' => { @@ -297,18 +353,20 @@ fn consume_field_type(tokens: &mut TokenIter) -> Vec { } fn consume_enum_discriminant(tokens: &mut TokenIter) -> Option { + let equal: Punct; match tokens.peek() { - Some(TokenTree::Punct(punct)) if punct.as_char() == '=' => (), + Some(TokenTree::Punct(punct)) if punct.as_char() == '=' => { + equal = punct.clone(); + } _ => return None, }; - let enum_discriminant_tokens = consume_stuff_until(tokens, |token| match token { - TokenTree::Punct(punct) if punct.as_char() == ',' => true, - _ => false, - }); + // consume '=' + tokens.next(); Some(EnumDiscriminant { - tokens: enum_discriminant_tokens, + _equal: equal, + expression: consume_expression(tokens), }) } @@ -551,6 +609,23 @@ fn parse_fn_params(tokens: TokenStream) -> Punctuated { fields } +#[doc(hidden)] +pub fn parse_expression_list(tokens: TokenStream) -> Punctuated { + let mut tokens = tokens.into_iter().peekable(); + let mut expressions = Punctuated::new(); + + while tokens.peek().is_some() { + let expression = consume_expression(&mut tokens); + let expression = expression; + + let comma = consume_comma(&mut tokens); + + expressions.push(expression, comma); + } + + expressions +} + // TODO - Return Result<...>, handle case where TokenStream is valid declaration, // but not a type or function. diff --git a/src/snapshots/venial__tests__parse_basic_expression.snap b/src/snapshots/venial__tests__parse_basic_expression.snap new file mode 100644 index 0000000..b363673 --- /dev/null +++ b/src/snapshots/venial__tests__parse_basic_expression.snap @@ -0,0 +1,14 @@ +--- +source: src/tests.rs +assertion_line: 592 +expression: expressions +--- +[ + [ + a, + "+", + b, + "+", + c, + ], +] diff --git a/src/snapshots/venial__tests__parse_empty_expression_list.snap b/src/snapshots/venial__tests__parse_empty_expression_list.snap new file mode 100644 index 0000000..d4ffbf7 --- /dev/null +++ b/src/snapshots/venial__tests__parse_empty_expression_list.snap @@ -0,0 +1,6 @@ +--- +source: src/tests.rs +assertion_line: 599 +expression: expressions +--- +[] diff --git a/src/snapshots/venial__tests__parse_enum_discriminant.snap b/src/snapshots/venial__tests__parse_enum_discriminant.snap index 1241ffc..d49d160 100644 --- a/src/snapshots/venial__tests__parse_enum_discriminant.snap +++ b/src/snapshots/venial__tests__parse_enum_discriminant.snap @@ -1,6 +1,6 @@ --- source: src/tests.rs -assertion_line: 445 +assertion_line: 514 expression: enum_type --- Enum( @@ -24,10 +24,15 @@ Enum( ), contents: Unit, discriminant: Some( - [ - "=", - 1, - ], + EnumDiscriminant { + _equal: Punct { + char: '=', + spacing: Alone, + }, + expression: [ + 1, + ], + }, ), }, EnumVariant { @@ -55,43 +60,48 @@ Enum( ], ), discriminant: Some( - [ - "=", - call, - ":", - ":", - some, - ":", - ":", - function, - Group { - delimiter: Parenthesis, - stream: TokenStream [ - Literal { - lit: 1, - }, - Punct { - char: ',', - spacing: Alone, - }, - Literal { - lit: 2, - }, - Punct { - char: ',', - spacing: Alone, - }, - Group { - delimiter: Brace, - stream: TokenStream [ - Literal { - lit: 3, - }, - ], - }, - ], + EnumDiscriminant { + _equal: Punct { + char: '=', + spacing: Alone, }, - ], + expression: [ + call, + ":", + ":", + some, + ":", + ":", + function, + Group { + delimiter: Parenthesis, + stream: TokenStream [ + Literal { + lit: 1, + }, + Punct { + char: ',', + spacing: Alone, + }, + Literal { + lit: 2, + }, + Punct { + char: ',', + spacing: Alone, + }, + Group { + delimiter: Brace, + stream: TokenStream [ + Literal { + lit: 3, + }, + ], + }, + ], + }, + ], + }, ), }, EnumVariant { @@ -133,30 +143,35 @@ Enum( ], ), discriminant: Some( - [ - "=", - A, - "<", - B, - ">", - Group { - delimiter: Parenthesis, - stream: TokenStream [ - Ident { - sym: c, - }, - ], - }, - "+", - Group { - delimiter: Brace, - stream: TokenStream [ - Ident { - sym: D, - }, - ], + EnumDiscriminant { + _equal: Punct { + char: '=', + spacing: Alone, }, - ], + expression: [ + A, + "<", + B, + ">", + Group { + delimiter: Parenthesis, + stream: TokenStream [ + Ident { + sym: c, + }, + ], + }, + "+", + Group { + delimiter: Brace, + stream: TokenStream [ + Ident { + sym: D, + }, + ], + }, + ], + }, ), }, ], diff --git a/src/snapshots/venial__tests__parse_expression_binary_or.snap b/src/snapshots/venial__tests__parse_expression_binary_or.snap new file mode 100644 index 0000000..079a159 --- /dev/null +++ b/src/snapshots/venial__tests__parse_expression_binary_or.snap @@ -0,0 +1,17 @@ +--- +source: src/tests.rs +assertion_line: 629 +expression: expressions +--- +[ + [ + a, + "|", + b, + ], + [ + c, + "|", + d, + ], +] diff --git a/src/snapshots/venial__tests__parse_expression_list_items.snap b/src/snapshots/venial__tests__parse_expression_list_items.snap new file mode 100644 index 0000000..d4c7e04 --- /dev/null +++ b/src/snapshots/venial__tests__parse_expression_list_items.snap @@ -0,0 +1,15 @@ +--- +source: src/tests.rs +assertion_line: 606 +expression: expressions +--- +[ + [ + 1, + "+", + 2, + ], + [ + 3, + ], +] diff --git a/src/snapshots/venial__tests__parse_expression_list_turbofish.snap b/src/snapshots/venial__tests__parse_expression_list_turbofish.snap new file mode 100644 index 0000000..b35cb8f --- /dev/null +++ b/src/snapshots/venial__tests__parse_expression_list_turbofish.snap @@ -0,0 +1,24 @@ +--- +source: src/tests.rs +assertion_line: 613 +expression: expressions +--- +[ + [ + a, + ":", + ":", + "<", + b, + "+", + c, + ",", + d, + ">", + "+", + e, + ], + [ + f, + ], +] diff --git a/src/tests.rs b/src/tests.rs index 293b866..8be7197 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,4 +1,7 @@ -use crate::{parse_declaration, Declaration, GenericParam, Struct, WhereClauseItem}; +use crate::{ + parse_declaration, parse_expression_list, Declaration, Expression, GenericParam, Punctuated, + Struct, WhereClauseItem, +}; use insta::assert_debug_snapshot; use proc_macro2::TokenStream; @@ -21,6 +24,15 @@ fn parse_declaration_checked(tokens: TokenStream) -> Declaration { declaration } +fn parse_expression_list_checked(tokens: TokenStream) -> Punctuated { + let initial_tokens = tokens.clone(); + let expression = parse_expression_list(tokens); + + similar_asserts::assert_str_eq!(quote!(#expression), initial_tokens); + + expression +} + // ============= // BASIC PARSING // ============= @@ -569,6 +581,54 @@ fn parse_macro_in_where_clause() { assert_debug_snapshot!(enum_type); } +// =========== +// EXPRESSIONS +// =========== + +#[test] +fn parse_basic_expression() { + let expressions = parse_expression_list_checked(quote!(a + b + c)); + + assert_debug_snapshot!(expressions); +} + +#[test] +fn parse_empty_expression_list() { + let expressions = parse_expression_list_checked(quote!()); + + assert_debug_snapshot!(expressions); +} + +#[test] +fn parse_expression_list_items() { + let expressions = parse_expression_list_checked(quote!(1 + 2, 3)); + + assert_debug_snapshot!(expressions); +} + +#[test] +fn parse_expression_list_turbofish() { + let expressions = parse_expression_list_checked(quote!(a:: + e, f)); + + assert_debug_snapshot!(expressions); +} + +// FIXME +#[test] +#[should_panic] +fn parse_closure_expression() { + let expressions = parse_expression_list_checked(quote!(|a, b| c)); + + assert_eq!(expressions.len(), 1); +} + +#[test] +fn parse_expression_binary_or() { + let expressions = parse_expression_list_checked(quote!(a | b, c | d)); + + assert_debug_snapshot!(expressions); +} + // ========= // FUNCTIONS // ========= diff --git a/src/types.rs b/src/types.rs index 38f4ddf..9c33e9a 100644 --- a/src/types.rs +++ b/src/types.rs @@ -5,6 +5,13 @@ use quote::{ToTokens, TokenStreamExt as _}; use crate::Punctuated; +// TODO - normalize naming convention of token fields + +#[derive(Clone)] +pub struct Expression { + pub tokens: Vec, +} + /// The declaration of a Rust type. /// /// **Example input:** @@ -336,9 +343,10 @@ pub struct TyExpr { /// B = (some + arbitrary.expression()), /// } /// ``` -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct EnumDiscriminant { - pub tokens: Vec, + pub _equal: Punct, + pub expression: Expression, } // --- Debug impls --- @@ -356,6 +364,16 @@ impl<'a> std::fmt::Debug for TokenRef<'a> { } } +impl std::fmt::Debug for Expression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut list = f.debug_list(); + for token in &self.tokens { + list.entry(&TokenRef(&token)); + } + list.finish() + } +} + impl std::fmt::Debug for TupleStructFields { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.fields.fmt(f) @@ -474,18 +492,16 @@ impl std::fmt::Debug for TyExpr { } } -impl std::fmt::Debug for EnumDiscriminant { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut list = f.debug_list(); +// --- ToTokens impls --- + +impl ToTokens for Expression { + fn to_tokens(&self, tokens: &mut TokenStream) { for token in &self.tokens { - list.entry(&TokenRef(&token)); + tokens.append(token.clone()); } - list.finish() } } -// --- ToTokens impls --- - impl ToTokens for Declaration { fn to_tokens(&self, tokens: &mut TokenStream) { match self { @@ -722,9 +738,8 @@ impl ToTokens for TyExpr { impl ToTokens for EnumDiscriminant { fn to_tokens(&self, tokens: &mut TokenStream) { - for token in &self.tokens { - tokens.append(token.clone()); - } + self._equal.to_tokens(tokens); + self.expression.to_tokens(tokens); } }