Skip to content

Commit

Permalink
drastically increase script loading speed via optimised token pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
@dfsek
dfsek committed Jan 6, 2021
1 parent 44176f7 commit 25ae2b3
Show file tree
Hide file tree
Showing 7 changed files with 92 additions and 163 deletions.
53 changes: 16 additions & 37 deletions common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
import com.dfsek.terra.api.structures.tokenizer.Position;
import com.dfsek.terra.api.structures.tokenizer.Token;
import com.dfsek.terra.api.structures.tokenizer.Tokenizer;
import com.dfsek.terra.api.structures.tokenizer.exceptions.TokenizerException;
import com.dfsek.terra.api.util.GlueList;

import java.util.Collections;
Expand Down Expand Up @@ -79,18 +78,7 @@ public String getID() {
* @throws ParseException If parsing fails.
*/
public Block parse() throws ParseException {
Tokenizer tokenizer = new Tokenizer(data);

TokenHolder tokens = new TokenHolder();
try {
Token t = tokenizer.fetch();
while(t != null) {
tokens.add(t);
t = tokenizer.fetch();
}
} catch(TokenizerException e) {
throw new ParseException("Failed to tokenize input", new Position(0, 0), e);
}
Tokenizer tokens = new Tokenizer(data);

// Parse ID
ParserUtil.checkType(tokens.consume(), Token.Type.ID); // First token must be ID
Expand All @@ -99,21 +87,12 @@ public Block parse() throws ParseException {
ParserUtil.checkType(tokens.consume(), Token.Type.STATEMENT_END);
this.id = idToken.getContent();

// Check for dangling brackets
int blockLevel = 0;
for(Token t : tokens.getTokens()) {
if(t.getType().equals(Token.Type.BLOCK_BEGIN)) blockLevel++;
else if(t.getType().equals(Token.Type.BLOCK_END)) blockLevel--;
if(blockLevel < 0) throw new ParseException("Dangling closing brace", t.getPosition());
}
if(blockLevel != 0)
throw new ParseException("Dangling opening brace", tokens.getTokens().get(tokens.getTokens().size() - 1).getPosition());

return parseBlock(tokens, new HashMap<>(), false);
}


private Keyword<?> parseLoopLike(TokenHolder tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
private Keyword<?> parseLoopLike(Tokenizer tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {

Token identifier = tokens.consume();
ParserUtil.checkType(identifier, Token.Type.IF_STATEMENT, Token.Type.WHILE_LOOP, Token.Type.FOR_LOOP);
Expand All @@ -132,7 +111,7 @@ private Keyword<?> parseLoopLike(TokenHolder tokens, Map<String, Variable<?>> va
}
}

private WhileKeyword parseWhileLoop(TokenHolder tokens, Map<String, Variable<?>> variableMap, Position start) throws ParseException {
private WhileKeyword parseWhileLoop(Tokenizer tokens, Map<String, Variable<?>> variableMap, Position start) throws ParseException {
Returnable<?> first = parseExpression(tokens, true, variableMap);
ParserUtil.checkReturnType(first, Returnable.ReturnType.BOOLEAN);

Expand All @@ -141,7 +120,7 @@ private WhileKeyword parseWhileLoop(TokenHolder tokens, Map<String, Variable<?>>
return new WhileKeyword(parseStatementBlock(tokens, variableMap, true), (Returnable<Boolean>) first, start); // While loop
}

private IfKeyword parseIfStatement(TokenHolder tokens, Map<String, Variable<?>> variableMap, Position start, boolean loop) throws ParseException {
private IfKeyword parseIfStatement(Tokenizer tokens, Map<String, Variable<?>> variableMap, Position start, boolean loop) throws ParseException {
Returnable<?> condition = parseExpression(tokens, true, variableMap);
ParserUtil.checkReturnType(condition, Returnable.ReturnType.BOOLEAN);

Expand All @@ -168,7 +147,7 @@ private IfKeyword parseIfStatement(TokenHolder tokens, Map<String, Variable<?>>
return new IfKeyword(statement, (Returnable<Boolean>) condition, elseIf, elseBlock, start); // If statement
}

private Block parseStatementBlock(TokenHolder tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
private Block parseStatementBlock(Tokenizer tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {

if(tokens.get().getType().equals(Token.Type.BLOCK_BEGIN)) {
ParserUtil.checkType(tokens.consume(), Token.Type.BLOCK_BEGIN);
Expand All @@ -183,7 +162,7 @@ private Block parseStatementBlock(TokenHolder tokens, Map<String, Variable<?>> v
}
}

private ForKeyword parseForLoop(TokenHolder tokens, Map<String, Variable<?>> old, Position start) throws ParseException {
private ForKeyword parseForLoop(Tokenizer tokens, Map<String, Variable<?>> old, Position start) throws ParseException {
Map<String, Variable<?>> variableMap = new HashMap<>(old); // New scope
Token f = tokens.get();
ParserUtil.checkType(f, Token.Type.NUMBER_VARIABLE, Token.Type.STRING_VARIABLE, Token.Type.BOOLEAN_VARIABLE, Token.Type.IDENTIFIER);
Expand Down Expand Up @@ -216,7 +195,7 @@ private ForKeyword parseForLoop(TokenHolder tokens, Map<String, Variable<?>> old
return new ForKeyword(parseStatementBlock(tokens, variableMap, true), initializer, (Returnable<Boolean>) conditional, incrementer, start);
}

private Returnable<?> parseExpression(TokenHolder tokens, boolean full, Map<String, Variable<?>> variableMap) throws ParseException {
private Returnable<?> parseExpression(Tokenizer tokens, boolean full, Map<String, Variable<?>> variableMap) throws ParseException {
boolean booleanInverted = false; // Check for boolean not operator
boolean negate = false;
if(tokens.get().getType().equals(Token.Type.BOOLEAN_NOT)) {
Expand Down Expand Up @@ -259,7 +238,7 @@ else if(variableMap.containsKey(id.getContent())) {
return expression;
}

private ConstantExpression<?> parseConstantExpression(TokenHolder tokens) throws ParseException {
private ConstantExpression<?> parseConstantExpression(Tokenizer tokens) throws ParseException {
Token constantToken = tokens.consume();
Position position = constantToken.getPosition();
switch(constantToken.getType()) {
Expand All @@ -275,15 +254,15 @@ private ConstantExpression<?> parseConstantExpression(TokenHolder tokens) throws
}
}

private Returnable<?> parseGroup(TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
private Returnable<?> parseGroup(Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
ParserUtil.checkType(tokens.consume(), Token.Type.GROUP_BEGIN);
Returnable<?> expression = parseExpression(tokens, true, variableMap); // Parse inside of group as a separate expression
ParserUtil.checkType(tokens.consume(), Token.Type.GROUP_END);
return expression;
}


private BinaryOperation<?, ?> parseBinaryOperation(Returnable<?> left, TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
private BinaryOperation<?, ?> parseBinaryOperation(Returnable<?> left, Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
Token binaryOperator = tokens.consume();
ParserUtil.checkBinaryOperator(binaryOperator);

Expand Down Expand Up @@ -337,7 +316,7 @@ private Returnable<?> parseGroup(TokenHolder tokens, Map<String, Variable<?>> va
}
}

private Variable<?> parseVariableDeclaration(TokenHolder tokens, Returnable.ReturnType type) throws ParseException {
private Variable<?> parseVariableDeclaration(Tokenizer tokens, Returnable.ReturnType type) throws ParseException {
ParserUtil.checkVarType(tokens.get(), type); // Check for type mismatch
switch(type) {
case NUMBER:
Expand All @@ -350,7 +329,7 @@ private Variable<?> parseVariableDeclaration(TokenHolder tokens, Returnable.Retu
throw new UnsupportedOperationException("Unsupported variable type: " + type);
}

private Block parseBlock(TokenHolder tokens, Map<String, Variable<?>> superVars, boolean loop) throws ParseException {
private Block parseBlock(Tokenizer tokens, Map<String, Variable<?>> superVars, boolean loop) throws ParseException {
List<Item<?>> parsedItems = new GlueList<>();

Map<String, Variable<?>> parsedVariables = new HashMap<>(superVars); // New hashmap as to not mutate parent scope's declarations.
Expand All @@ -366,7 +345,7 @@ private Block parseBlock(TokenHolder tokens, Map<String, Variable<?>> superVars,
return new Block(parsedItems, first.getPosition());
}

private Item<?> parseItem(TokenHolder tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
private Item<?> parseItem(Tokenizer tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
Token token = tokens.get();
if(loop) ParserUtil.checkType(token, Token.Type.IDENTIFIER, Token.Type.IF_STATEMENT, Token.Type.WHILE_LOOP, Token.Type.FOR_LOOP,
Token.Type.NUMBER_VARIABLE, Token.Type.STRING_VARIABLE, Token.Type.BOOLEAN_VARIABLE, Token.Type.RETURN, Token.Type.BREAK, Token.Type.CONTINUE, Token.Type.FAIL);
Expand Down Expand Up @@ -401,7 +380,7 @@ private Item<?> parseItem(TokenHolder tokens, Map<String, Variable<?>> variableM
else throw new UnsupportedOperationException("Unexpected token " + token.getType() + ": " + token.getPosition());
}

private Assignment<?> parseAssignment(Variable<?> variable, TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
private Assignment<?> parseAssignment(Variable<?> variable, Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
Token name = tokens.get();

ParserUtil.checkType(tokens.consume(), Token.Type.IDENTIFIER);
Expand All @@ -415,7 +394,7 @@ private Assignment<?> parseAssignment(Variable<?> variable, TokenHolder tokens,
return new Assignment<>((Variable<Object>) variable, (Returnable<Object>) expression, name.getPosition());
}

private Function<?> parseFunction(TokenHolder tokens, boolean fullStatement, Map<String, Variable<?>> variableMap) throws ParseException {
private Function<?> parseFunction(Tokenizer tokens, boolean fullStatement, Map<String, Variable<?>> variableMap) throws ParseException {
Token identifier = tokens.consume();
ParserUtil.checkType(identifier, Token.Type.IDENTIFIER); // First token must be identifier

Expand Down Expand Up @@ -449,7 +428,7 @@ private Function<?> parseFunction(TokenHolder tokens, boolean fullStatement, Map
}


private List<Returnable<?>> getArgs(TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
private List<Returnable<?>> getArgs(Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
List<Returnable<?>> args = new GlueList<>();

while(!tokens.get().getType().equals(Token.Type.GROUP_END)) {
Expand Down
59 changes: 0 additions & 59 deletions common/src/main/java/com/dfsek/terra/api/structures/parser/TokenHolder.java
View file

This file was deleted.

63 changes: 57 additions & 6 deletions common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,74 @@
package com.dfsek.terra.api.structures.tokenizer;

import com.dfsek.terra.api.structures.parser.exceptions.ParseException;
import com.dfsek.terra.api.structures.tokenizer.exceptions.EOFException;
import com.dfsek.terra.api.structures.tokenizer.exceptions.FormatException;
import com.dfsek.terra.api.structures.tokenizer.exceptions.TokenizerException;
import com.google.common.collect.Sets;

import java.io.StringReader;
import java.util.Set;
import java.util.Stack;

public class Tokenizer {
public static final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', '=', '{', '}', '+', '-', '*', '/', '>', '<', '!'); // Reserved chars
private final Lookahead reader;
private final Stack<Token> brackets = new Stack<>();
private Token current;

public static final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', '=', '{', '}', '+', '-', '*', '/', '>', '<', '!'); // Reserved chars
public Tokenizer(String data) throws ParseException {
reader = new Lookahead(new StringReader(data + '\0'));
current = fetchCheck();
}

/**
* Get the first token.
*
* @return First token
* @throws ParseException If token does not exist
*/
public Token get() throws ParseException {
if(!hasNext()) throw new ParseException("Unexpected end of input", current.getPosition());
return current;
}

public Tokenizer(String data) {
reader = new Lookahead(new StringReader(data + '\0'));
/**
* Consume (get and remove) the first token.
*
* @return First token
* @throws ParseException If token does not exist
*/
public Token consume() throws ParseException {
if(!hasNext()) throw new ParseException("Unexpected end of input", current.getPosition());
Token temp = current;
current = fetchCheck();
return temp;
}

/**
* Whether this {@code Tokenizer} contains additional tokens.
*
* @return {@code true} if more tokens are present, otherwise {@code false}
*/
public boolean hasNext() {
return !(current == null);
}

private Token fetchCheck() throws ParseException {
Token fetch = fetch();
if(fetch != null) {
if(fetch.getType().equals(Token.Type.BLOCK_BEGIN)) brackets.push(fetch); // Opening bracket
else if(fetch.getType().equals(Token.Type.BLOCK_END)) {
if(!brackets.isEmpty()) brackets.pop();
else throw new ParseException("Dangling opening brace", new Position(0, 0));
}
} else if(!brackets.isEmpty()) {
throw new ParseException("Dangling closing brace", brackets.peek().getPosition());
}
return fetch;
}

public Token fetch() throws TokenizerException {
private Token fetch() throws TokenizerException {
while(!reader.current().isEOF() && reader.current().isWhitespace()) reader.consume();

while(reader.matches("//", true)) skipLine(); // Skip line if comment
Expand Down Expand Up @@ -66,7 +116,7 @@ public Token fetch() throws TokenizerException {
continue;
} else ignoreNext = false;
if(reader.current().isEOF())
throw new FormatException("No end of string literal found. " + reader.getLine() + ":" + reader.getIndex());
throw new FormatException("No end of string literal found. ", new Position(reader.getLine(), reader.getIndex()));
string.append(reader.consume());
}
reader.consume(); // Consume last quote
Expand Down Expand Up @@ -166,14 +216,15 @@ private void consumeWhitespace() {
}

private void skipTo(String s) throws EOFException {
Position begin = new Position(reader.getLine(), reader.getIndex());
while(!reader.current().isEOF()) {
if(reader.matches(s, true)) {
consumeWhitespace();
return;
}
reader.consume();
}
throw new EOFException("No end of expression found.");
throw new EOFException("No end of expression found.", begin);
}

public boolean isSyntaxSignificant(char c) {
Expand Down
18 changes: 6 additions & 12 deletions common/src/main/java/com/dfsek/terra/api/structures/tokenizer/exceptions/EOFException.java
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
package com.dfsek.terra.api.structures.tokenizer.exceptions;

public class EOFException extends TokenizerException {

public EOFException(String s) {
super(s);
}
import com.dfsek.terra.api.structures.tokenizer.Position;

public EOFException() {
super();
}
public class EOFException extends TokenizerException {

public EOFException(String message, Throwable cause) {
super(message, cause);
public EOFException(String message, Position position) {
super(message, position);
}

public EOFException(Throwable cause) {
super(cause);
public EOFException(String message, Position position, Throwable cause) {
super(message, position, cause);
}
}
18 changes: 6 additions & 12 deletions ...on/src/main/java/com/dfsek/terra/api/structures/tokenizer/exceptions/FormatException.java
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,14 @@
package com.dfsek.terra.api.structures.tokenizer.exceptions;

public class FormatException extends TokenizerException {

public FormatException(String s) {
super(s);
}
import com.dfsek.terra.api.structures.tokenizer.Position;

public FormatException() {
super();
}
public class FormatException extends TokenizerException {

public FormatException(String message, Throwable cause) {
super(message, cause);
public FormatException(String message, Position position) {
super(message, position);
}

public FormatException(Throwable cause) {
super(cause);
public FormatException(String message, Position position, Throwable cause) {
super(message, position, cause);
}
}
Loading

0 comments on commit 25ae2b3

Please sign in to comment.