Skip to content

Commit

Permalink
Add basic expression parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@PoignardAzur
PoignardAzur committed Apr 11, 2022
1 parent d317dda commit 12cbe05
Show file tree
Hide file tree
Showing 10 changed files with 335 additions and 94 deletions.
6 changes: 3 additions & 3 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,10 @@ mod types;
mod types_edition;

pub use error::Error;
pub use parse::parse_declaration;
pub use parse::{parse_declaration, parse_expression_list};
pub use punctuated::Punctuated;
pub use types::{
Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, GenericBound, GenericParam,
GenericParams, NamedField, NamedStructFields, Struct, StructFields, TupleField,
Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, Expression, GenericBound,
GenericParam, GenericParams, NamedField, NamedStructFields, Struct, StructFields, TupleField,
TupleStructFields, TyExpr, Union, VisMarker, WhereClause, WhereClauseItem,
};
103 changes: 89 additions & 14 deletions src/parse.rs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
use crate::punctuated::Punctuated;
use crate::types::{
Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, Function, FunctionParameter,
FunctionQualifiers, GenericBound, GenericParam, GenericParams, NamedField, NamedStructFields,
Struct, StructFields, TupleField, TupleStructFields, TyExpr, Union, VisMarker, WhereClause,
WhereClauseItem,
Attribute, Declaration, Enum, EnumDiscriminant, EnumVariant, Expression, Function,
FunctionParameter, FunctionQualifiers, GenericBound, GenericParam, GenericParams, NamedField,
NamedStructFields, Struct, StructFields, TupleField, TupleStructFields, TyExpr, Union,
VisMarker, WhereClause, WhereClauseItem,
};
use proc_macro2::{Delimiter, Group, Ident, Punct, TokenStream, TokenTree};
use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, TokenStream, TokenTree};
use std::iter::Peekable;

type TokenIter = Peekable<proc_macro2::token_stream::IntoIter>;
Expand Down Expand Up @@ -85,7 +85,7 @@ fn consume_declaration_name(tokens: &mut TokenIter) -> Ident {

// Consumes tokens until a separator is reached *unless* the
// separator in between angle brackets
// eg consume_stuff_until(..., ',') will consume all
// eg consume_stuff_until(..., |token| token == ',') will consume all
// of `Foobar<A, B>,` except for the last comma
pub(crate) fn consume_stuff_until(
tokens: &mut TokenIter,
Expand All @@ -96,8 +96,6 @@ pub(crate) fn consume_stuff_until(
let mut predicate = predicate;
let mut prev_token_is_dash = false;

// TODO - handle closures

loop {
let token = tokens.peek();
prev_token_is_dash = match &token {
Expand Down Expand Up @@ -131,6 +129,64 @@ pub(crate) fn consume_stuff_until(
output_tokens
}

// Consumes tokens until a comma is reached, except in
// various corner cases related to expression syntax.
// eg consume_expression(...) will consume all
// of `a + |b, c| d, e::<F, G>(), h,` except for the last comma
pub(crate) fn consume_expression(tokens: &mut TokenIter) -> Expression {
let mut output_tokens = Vec::new();

// TODO - handle closures
// TODO - handle <T as Trait>::xxx syntax

// TODO - use matches instead?
#[derive(Debug, PartialEq)]
enum PrevToken {
Any,
FirstColon,
DoubleColon,
}
let mut prev_token = PrevToken::Any;

loop {
let token = tokens.peek();
prev_token = match &token {
Some(TokenTree::Punct(punct))
if punct.as_char() == ':' && punct.spacing() == Spacing::Joint =>
{
output_tokens.push(tokens.next().unwrap());
PrevToken::FirstColon
}
Some(TokenTree::Punct(punct))
if punct.as_char() == ':' && prev_token == PrevToken::FirstColon =>
{
output_tokens.push(tokens.next().unwrap());
PrevToken::DoubleColon
}

Some(TokenTree::Punct(punct))
if punct.as_char() == '<' && prev_token == PrevToken::DoubleColon =>
{
let mut turbofish_contents = consume_stuff_until(tokens, |_| true);
output_tokens.append(&mut turbofish_contents);
PrevToken::Any
}

Some(TokenTree::Punct(punct)) if punct.as_char() == ',' => break,
None => break,

_ => {
output_tokens.push(tokens.next().unwrap());
PrevToken::Any
}
};
}

Expression {
tokens: output_tokens,
}
}

fn consume_comma(tokens: &mut TokenIter) -> Option<Punct> {
match tokens.peek() {
Some(TokenTree::Punct(punct)) if punct.as_char() == ',' => {
Expand Down Expand Up @@ -297,18 +353,20 @@ fn consume_field_type(tokens: &mut TokenIter) -> Vec<TokenTree> {
}

fn consume_enum_discriminant(tokens: &mut TokenIter) -> Option<EnumDiscriminant> {
let equal: Punct;
match tokens.peek() {
Some(TokenTree::Punct(punct)) if punct.as_char() == '=' => (),
Some(TokenTree::Punct(punct)) if punct.as_char() == '=' => {
equal = punct.clone();
}
_ => return None,
};

let enum_discriminant_tokens = consume_stuff_until(tokens, |token| match token {
TokenTree::Punct(punct) if punct.as_char() == ',' => true,
_ => false,
});
// consume '='
tokens.next();

Some(EnumDiscriminant {
tokens: enum_discriminant_tokens,
_equal: equal,
expression: consume_expression(tokens),
})
}

Expand Down Expand Up @@ -551,6 +609,23 @@ fn parse_fn_params(tokens: TokenStream) -> Punctuated<FunctionParameter> {
fields
}

#[doc(hidden)]
pub fn parse_expression_list(tokens: TokenStream) -> Punctuated<Expression> {
let mut tokens = tokens.into_iter().peekable();
let mut expressions = Punctuated::new();

while tokens.peek().is_some() {
let expression = consume_expression(&mut tokens);
let expression = expression;

let comma = consume_comma(&mut tokens);

expressions.push(expression, comma);
}

expressions
}

// TODO - Return Result<...>, handle case where TokenStream is valid declaration,
// but not a type or function.

Expand Down
14 changes: 14 additions & 0 deletions src/snapshots/venial__tests__parse_basic_expression.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
source: src/tests.rs
assertion_line: 592
expression: expressions
---
[
[
a,
"+",
b,
"+",
c,
],
]
6 changes: 6 additions & 0 deletions src/snapshots/venial__tests__parse_empty_expression_list.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
source: src/tests.rs
assertion_line: 599
expression: expressions
---
[]
143 changes: 79 additions & 64 deletions src/snapshots/venial__tests__parse_enum_discriminant.snap
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
source: src/tests.rs
assertion_line: 445
assertion_line: 514
expression: enum_type
---
Enum(
Expand All @@ -24,10 +24,15 @@ Enum(
),
contents: Unit,
discriminant: Some(
[
"=",
1,
],
EnumDiscriminant {
_equal: Punct {
char: '=',
spacing: Alone,
},
expression: [
1,
],
},
),
},
EnumVariant {
Expand Down Expand Up @@ -55,43 +60,48 @@ Enum(
],
),
discriminant: Some(
[
"=",
call,
":",
":",
some,
":",
":",
function,
Group {
delimiter: Parenthesis,
stream: TokenStream [
Literal {
lit: 1,
},
Punct {
char: ',',
spacing: Alone,
},
Literal {
lit: 2,
},
Punct {
char: ',',
spacing: Alone,
},
Group {
delimiter: Brace,
stream: TokenStream [
Literal {
lit: 3,
},
],
},
],
EnumDiscriminant {
_equal: Punct {
char: '=',
spacing: Alone,
},
],
expression: [
call,
":",
":",
some,
":",
":",
function,
Group {
delimiter: Parenthesis,
stream: TokenStream [
Literal {
lit: 1,
},
Punct {
char: ',',
spacing: Alone,
},
Literal {
lit: 2,
},
Punct {
char: ',',
spacing: Alone,
},
Group {
delimiter: Brace,
stream: TokenStream [
Literal {
lit: 3,
},
],
},
],
},
],
},
),
},
EnumVariant {
Expand Down Expand Up @@ -133,30 +143,35 @@ Enum(
],
),
discriminant: Some(
[
"=",
A,
"<",
B,
">",
Group {
delimiter: Parenthesis,
stream: TokenStream [
Ident {
sym: c,
},
],
},
"+",
Group {
delimiter: Brace,
stream: TokenStream [
Ident {
sym: D,
},
],
EnumDiscriminant {
_equal: Punct {
char: '=',
spacing: Alone,
},
],
expression: [
A,
"<",
B,
">",
Group {
delimiter: Parenthesis,
stream: TokenStream [
Ident {
sym: c,
},
],
},
"+",
Group {
delimiter: Brace,
stream: TokenStream [
Ident {
sym: D,
},
],
},
],
},
),
},
],
Expand Down
17 changes: 17 additions & 0 deletions src/snapshots/venial__tests__parse_expression_binary_or.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
source: src/tests.rs
assertion_line: 629
expression: expressions
---
[
[
a,
"|",
b,
],
[
c,
"|",
d,
],
]
15 changes: 15 additions & 0 deletions src/snapshots/venial__tests__parse_expression_list_items.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---
source: src/tests.rs
assertion_line: 606
expression: expressions
---
[
[
1,
"+",
2,
],
[
3,
],
]
Loading

0 comments on commit 12cbe05

Please sign in to comment.