danog/parser
1
0
Fork 0
mirror of https://github.com/danog/parser.git synced 2024-11-30 04:29:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: make parser, and lexer immutable

Browse Source
This commit is contained in:
Saif Eddin Gmati 2022-12-01 03:49:51 +01:00 committed by Ryan Chandler
parent bdef34b4b4
commit 77ff227bc0
19 changed files with 1428 additions and 1360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/snapshot.rs
View File

@ -3,6 +3,9 @@ use std::env;
use std::fs::read_dir;
use std::path::PathBuf;
static PARSER: Parser = Parser::new();
static LEXER: Lexer = Lexer::new();
fn main() {
let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
let mut entries = read_dir(manifest.join("tests"))
@ -14,10 +17,6 @@ fn main() {
entries.sort();
let mut content = String::new();
content.push_str("/// this file is auto-generated by the build script.\n");
content.push_str("/// you should never manually change it.\n\n\n");
for entry in entries {
let code_filename = entry.join("code.php");
let ast_filename = entry.join("ast.txt");
@ -46,8 +45,7 @@ fn main() {
}
let code = std::fs::read_to_string(&code_filename).unwrap();
let mut lexer = Lexer::new();
let tokens = lexer.tokenize(code.as_bytes());
let tokens = LEXER.tokenize(code.as_bytes());
match tokens {
Ok(tokens) => {
@ -57,8 +55,7 @@ fn main() {
entry.to_string_lossy()
);
let mut parser = Parser::new(None);
let ast = parser.parse(tokens);
let ast = PARSER.parse(tokens);
match ast {
Ok(ast) => {
std::fs::write(ast_filename, format!("{:#?}\n", ast)).unwrap();

32
build.rs
View File

@ -26,7 +26,11 @@ fn main() {
let mut content = String::new();
content.push_str("/// this file is auto-generated by the build script.\n");
content.push_str("/// you should never manually change it.\n\n\n");
content.push_str("/// you should never manually change it.\n\n");
content.push_str("use php_parser_rs::prelude::{Lexer, Parser};\n");
content.push_str("use pretty_assertions::assert_str_eq;\n\n");
content.push_str("static PARSER: Parser = Parser::new();\n");
content.push_str("static LEXER: Lexer = Lexer::new();\n\n");
for entry in entries {
let code_filename = entry.join("code.php");
@ -98,10 +102,6 @@ fn build_success_test(
format!(
r#"#[test]
fn test_success_{}() {{
use php_parser_rs::prelude::Parser;
use php_parser_rs::prelude::Lexer;
use pretty_assertions::assert_str_eq;
let code_filename = "{}";
let ast_filename = "{}";
let tokens_filename = "{}";
@ -110,13 +110,11 @@ fn test_success_{}() {{
let expected_ast = std::fs::read_to_string(&ast_filename).unwrap();
let expected_tokens = std::fs::read_to_string(&tokens_filename).unwrap();
let mut lexer = Lexer::new();
let tokens = lexer.tokenize(code.as_bytes()).unwrap();
let tokens = LEXER.tokenize(code.as_bytes()).unwrap();
assert_str_eq!(expected_tokens.trim(), format!("{{:#?}}", tokens));
let mut parser = Parser::new(None);
let ast = parser.parse(tokens).unwrap();
let ast = PARSER.parse(tokens).unwrap();
assert_str_eq!(expected_ast.trim(), format!("{{:#?}}", ast));
}}
@ -137,17 +135,13 @@ fn build_lexer_error_test(
format!(
r#"#[test]
fn test_lexer_error_{}() {{
use php_parser_rs::prelude::Lexer;
use pretty_assertions::assert_str_eq;
let code_filename = "{}";
let lexer_error_filename = "{}";
let code = std::fs::read_to_string(&code_filename).unwrap();
let expected_error = std::fs::read_to_string(&lexer_error_filename).unwrap();
let mut lexer = Lexer::new();
let error = lexer.tokenize(code.as_bytes()).err().unwrap();
let error = LEXER.tokenize(code.as_bytes()).err().unwrap();
assert_str_eq!(
expected_error.trim(),
@ -171,10 +165,6 @@ fn build_parser_error_test(
format!(
r#"#[test]
fn test_paser_error_{}() {{
use php_parser_rs::prelude::Parser;
use php_parser_rs::prelude::Lexer;
use pretty_assertions::assert_str_eq;
let code_filename = "{}";
let tokens_filename = "{}";
let parser_error_filename = "{}";
@ -183,13 +173,11 @@ fn test_paser_error_{}() {{
let expected_tokens = std::fs::read_to_string(&tokens_filename).unwrap();
let expected_error = std::fs::read_to_string(&parser_error_filename).unwrap();
let mut lexer = Lexer::new();
let tokens = lexer.tokenize(code.as_bytes()).unwrap();
let tokens = LEXER.tokenize(code.as_bytes()).unwrap();
assert_str_eq!(expected_tokens.trim(), format!("{{:#?}}", tokens));
let mut parser = Parser::new(None);
let error = parser.parse(tokens).err().unwrap();
let error = PARSER.parse(tokens).err().unwrap();
assert_str_eq!(
expected_error.trim(),

657
src/lexer/mod.rs
View File

File diff suppressed because it is too large Load Diff

80
src/lexer/state.rs Normal file
View File

@ -0,0 +1,80 @@
use crate::lexer::token::Span;
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum StackState {
Initial,
Scripting,
Halted,
DoubleQuote,
LookingForVarname,
LookingForProperty,
VarOffset,
}
// TODO(azjezz): make `chars` a `[u8, N]`, and `State`, `State<const N: usize>`
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct State {
pub stack: Vec<StackState>,
pub chars: Vec<u8>,
pub cursor: usize,
pub current: Option<u8>,
pub span: Span,
}
impl State {
pub fn new<B: ?Sized + AsRef<[u8]>>(input: &B) -> Self {
let chars = input.as_ref().to_vec();
let current = chars.first().copied();
Self {
stack: vec![StackState::Initial],
chars,
current,
cursor: 0,
span: (1, 1),
}
}
pub fn enter_state(&mut self, state: StackState) {
*self.stack.last_mut().unwrap() = state;
}
pub fn push_state(&mut self, state: StackState) {
self.stack.push(state);
}
pub fn pop_state(&mut self) {
self.stack.pop();
}
pub fn peek_buf(&self) -> &[u8] {
&self.chars[self.cursor..]
}
pub fn peek_byte(&self, delta: usize) -> Option<u8> {
self.chars.get(self.cursor + delta).copied()
}
pub fn try_read(&self, search: &'static [u8]) -> bool {
self.peek_buf().starts_with(search)
}
pub fn skip(&mut self, count: usize) {
for _ in 0..count {
self.next();
}
}
pub fn next(&mut self) {
match self.current {
Some(b'\n') => {
self.span.0 += 1;
self.span.1 = 1;
}
Some(_) => self.span.1 += 1,
_ => {}
}
self.cursor += 1;
self.current = self.chars.get(self.cursor).copied();
}
}

4
src/main.rs
View File

@ -20,7 +20,7 @@ fn main() {
}
};
let mut lexer = Lexer::new();
let lexer = Lexer::new();
let tokens = match lexer.tokenize(contents.as_bytes()) {
Ok(tokens) => tokens,
Err(error) => {
@ -30,7 +30,7 @@ fn main() {
}
};
let mut parser = Parser::new(None);
let parser = Parser::new();
let ast = match parser.parse(tokens) {
Ok(ast) => ast,
Err(error) => {

15
src/parser/block.rs
View File

@ -1,17 +1,22 @@
use crate::lexer::token::TokenKind;
use crate::parser::ast::Block;
use crate::parser::error::ParseResult;
use crate::parser::state::State;
use crate::parser::Parser;
impl Parser {
pub(in crate::parser) fn block(&mut self, until: &TokenKind) -> ParseResult<Block> {
self.skip_comments();
pub(in crate::parser) fn block(
&self,
state: &mut State,
until: &TokenKind,
) -> ParseResult<Block> {
state.skip_comments();
let mut block = Block::new();
while !self.is_eof() && &self.current.kind != until {
block.push(self.statement()?);
self.skip_comments();
while !state.is_eof() && &state.current.kind != until {
block.push(self.statement(state)?);
state.skip_comments();
}
Ok(block)

176
src/parser/classish.rs
View File

@ -6,49 +6,50 @@ use crate::parser::ast::Expression;
use crate::parser::ast::Identifier;
use crate::parser::ast::Statement;
use crate::parser::error::ParseResult;
use crate::parser::state::State;
use crate::parser::Parser;
use crate::expect_token;
use crate::expected_token_err;
impl Parser {
pub(in crate::parser) fn class_definition(&mut self) -> ParseResult<Statement> {
let flags: Vec<ClassFlag> = self.class_flags()?.iter().map(|f| f.into()).collect();
pub(in crate::parser) fn class_definition(&self, state: &mut State) -> ParseResult<Statement> {
let flags: Vec<ClassFlag> = self.class_flags(state)?.iter().map(|f| f.into()).collect();
expect_token!([TokenKind::Class], self, ["`class`"]);
expect_token!([TokenKind::Class], state, ["`class`"]);
let name = self.ident()?;
let name = self.ident(state)?;
let mut extends: Option<Identifier> = None;
if self.current.kind == TokenKind::Extends {
self.next();
extends = Some(self.full_name()?.into());
if state.current.kind == TokenKind::Extends {
state.next();
extends = Some(self.full_name(state)?.into());
}
let implements = if self.current.kind == TokenKind::Implements {
self.next();
let implements = if state.current.kind == TokenKind::Implements {
state.next();
self.at_least_one_comma_separated::<Identifier>(&|parser| {
Ok(parser.full_name()?.into())
self.at_least_one_comma_separated::<Identifier>(state, &|parser, state| {
Ok(parser.full_name(state)?.into())
})?
} else {
Vec::new()
};
self.lbrace()?;
self.lbrace(state)?;
let mut body = Vec::new();
while self.current.kind != TokenKind::RightBrace {
self.gather_comments();
while state.current.kind != TokenKind::RightBrace {
state.gather_comments();
if self.current.kind == TokenKind::RightBrace {
self.clear_comments();
if state.current.kind == TokenKind::RightBrace {
state.clear_comments();
break;
}
body.push(self.class_statement(flags.clone())?);
body.push(self.class_statement(state, flags.clone())?);
}
self.rbrace()?;
self.rbrace(state)?;
Ok(Statement::Class {
name: name.into(),
@ -59,35 +60,38 @@ impl Parser {
})
}
pub(in crate::parser) fn interface_definition(&mut self) -> ParseResult<Statement> {
expect_token!([TokenKind::Interface], self, ["`interface`"]);
pub(in crate::parser) fn interface_definition(
&self,
state: &mut State,
) -> ParseResult<Statement> {
expect_token!([TokenKind::Interface], state, ["`interface`"]);
let name = self.ident()?;
let name = self.ident(state)?;
let extends = if self.current.kind == TokenKind::Extends {
self.next();
let extends = if state.current.kind == TokenKind::Extends {
state.next();
self.at_least_one_comma_separated::<Identifier>(&|parser| {
Ok(parser.full_name()?.into())
self.at_least_one_comma_separated::<Identifier>(state, &|parser, state| {
Ok(parser.full_name(state)?.into())
})?
} else {
Vec::new()
};
self.lbrace()?;
self.lbrace(state)?;
let mut body = Vec::new();
while self.current.kind != TokenKind::RightBrace && !self.is_eof() {
self.gather_comments();
while state.current.kind != TokenKind::RightBrace && !state.is_eof() {
state.gather_comments();
if self.current.kind == TokenKind::RightBrace {
self.clear_comments();
if state.current.kind == TokenKind::RightBrace {
state.clear_comments();
break;
}
body.push(self.interface_statement()?);
body.push(self.interface_statement(state)?);
}
self.rbrace()?;
self.rbrace(state)?;
Ok(Statement::Interface {
name: name.into(),
@ -96,25 +100,25 @@ impl Parser {
})
}
pub(in crate::parser) fn trait_definition(&mut self) -> ParseResult<Statement> {
expect_token!([TokenKind::Trait], self, ["`trait`"]);
pub(in crate::parser) fn trait_definition(&self, state: &mut State) -> ParseResult<Statement> {
expect_token!([TokenKind::Trait], state, ["`trait`"]);
let name = self.ident()?;
let name = self.ident(state)?;
self.lbrace()?;
self.lbrace(state)?;
let mut body = Vec::new();
while self.current.kind != TokenKind::RightBrace && !self.is_eof() {
self.gather_comments();
while state.current.kind != TokenKind::RightBrace && !state.is_eof() {
state.gather_comments();
if self.current.kind == TokenKind::RightBrace {
self.clear_comments();
if state.current.kind == TokenKind::RightBrace {
state.clear_comments();
break;
}
body.push(self.trait_statement()?);
body.push(self.trait_statement(state)?);
}
self.rbrace()?;
self.rbrace(state)?;
Ok(Statement::Trait {
name: name.into(),
@ -122,47 +126,50 @@ impl Parser {
})
}
pub(in crate::parser) fn anonymous_class_definition(&mut self) -> ParseResult<Expression> {
self.next();
pub(in crate::parser) fn anonymous_class_definition(
&self,
state: &mut State,
) -> ParseResult<Expression> {
state.next();
expect_token!([TokenKind::Class], self, ["`class`"]);
expect_token!([TokenKind::Class], state, ["`class`"]);
let mut args = vec![];
if self.current.kind == TokenKind::LeftParen {
self.lparen()?;
if state.current.kind == TokenKind::LeftParen {
self.lparen(state)?;
args = self.args_list()?;
args = self.args_list(state)?;
self.rparen()?;
self.rparen(state)?;
}
let mut extends: Option<Identifier> = None;
if self.current.kind == TokenKind::Extends {
self.next();
extends = Some(self.full_name()?.into());
if state.current.kind == TokenKind::Extends {
state.next();
extends = Some(self.full_name(state)?.into());
}
let mut implements = Vec::new();
if self.current.kind == TokenKind::Implements {
self.next();
if state.current.kind == TokenKind::Implements {
state.next();
while self.current.kind != TokenKind::LeftBrace {
self.optional_comma()?;
while state.current.kind != TokenKind::LeftBrace {
self.optional_comma(state)?;
implements.push(self.full_name()?.into());
implements.push(self.full_name(state)?.into());
}
}
self.lbrace()?;
self.lbrace(state)?;
let mut body = Vec::new();
while self.current.kind != TokenKind::RightBrace && !self.is_eof() {
body.push(self.anonymous_class_statement()?);
while state.current.kind != TokenKind::RightBrace && !state.is_eof() {
body.push(self.anonymous_class_statement(state)?);
}
self.rbrace()?;
self.rbrace(state)?;
Ok(Expression::New {
target: Box::new(Expression::AnonymousClass {
@ -174,17 +181,17 @@ impl Parser {
})
}
pub(in crate::parser) fn enum_definition(&mut self) -> ParseResult<Statement> {
self.next();
pub(in crate::parser) fn enum_definition(&self, state: &mut State) -> ParseResult<Statement> {
state.next();
let name = self.ident()?;
let name = self.ident(state)?;
let backed_type: Option<BackedEnumType> = if self.current.kind == TokenKind::Colon {
self.colon()?;
let backed_type: Option<BackedEnumType> = if state.current.kind == TokenKind::Colon {
self.colon(state)?;
match self.current.kind.clone() {
match state.current.kind.clone() {
TokenKind::Identifier(s) if s == b"string" || s == b"int" => {
self.next();
state.next();
Some(match &s[..] {
b"string" => BackedEnumType::String,
@ -193,7 +200,7 @@ impl Parser {
})
}
_ => {
return expected_token_err!(["`string`", "`int`"], self);
return expected_token_err!(["`string`", "`int`"], state);
}
}
} else {
@ -201,25 +208,25 @@ impl Parser {
};
let mut implements = Vec::new();
if self.current.kind == TokenKind::Implements {
self.next();
if state.current.kind == TokenKind::Implements {
state.next();
while self.current.kind != TokenKind::LeftBrace {
implements.push(self.full_name()?.into());
while state.current.kind != TokenKind::LeftBrace {
implements.push(self.full_name(state)?.into());
self.optional_comma()?;
self.optional_comma(state)?;
}
}
self.lbrace()?;
self.lbrace(state)?;
let mut body = Block::new();
while self.current.kind != TokenKind::RightBrace {
self.skip_comments();
body.push(self.enum_statement(backed_type.is_some())?);
while state.current.kind != TokenKind::RightBrace {
state.skip_comments();
body.push(self.enum_statement(state, backed_type.is_some())?);
}
self.rbrace()?;
self.rbrace(state)?;
match backed_type {
Some(backed_type) => Ok(Statement::BackedEnum {
@ -237,17 +244,18 @@ impl Parser {
}
fn at_least_one_comma_separated<T>(
&mut self,
func: &(dyn Fn(&mut Parser) -> ParseResult<T>),
&self,
state: &mut State,
func: &(dyn Fn(&Parser, &mut State) -> ParseResult<T>),
) -> ParseResult<Vec<T>> {
let mut result: Vec<T> = vec![];
loop {
result.push(func(self)?);
if self.current.kind != TokenKind::Comma {
result.push(func(self, state)?);
if state.current.kind != TokenKind::Comma {
break;
}
self.next();
state.next();
}
Ok(result)

225
src/parser/classish_statement.rs
View File

@ -7,6 +7,7 @@ use crate::parser::ast::TraitAdaptation;
use crate::parser::error::ParseError;
use crate::parser::error::ParseResult;
use crate::parser::precedence::Precedence;
use crate::parser::state::State;
use crate::parser::Parser;
use crate::expect_token;
@ -24,121 +25,136 @@ pub enum ClassishDefinitionType {
impl Parser {
pub(in crate::parser) fn class_statement(
&mut self,
&self,
state: &mut State,
flags: Vec<ClassFlag>,
) -> ParseResult<Statement> {
self.complete_class_statement(ClassishDefinitionType::Class(flags))
self.complete_class_statement(state, ClassishDefinitionType::Class(flags))
}
pub(in crate::parser) fn interface_statement(&mut self) -> ParseResult<Statement> {
if self.current.kind == TokenKind::Const {
return self.parse_classish_const(vec![]);
pub(in crate::parser) fn interface_statement(
&self,
state: &mut State,
) -> ParseResult<Statement> {
if state.current.kind == TokenKind::Const {
return self.parse_classish_const(state, vec![]);
}
if self.current.kind == TokenKind::Function {
return self.method(ClassishDefinitionType::Interface, vec![]);
if state.current.kind == TokenKind::Function {
return self.method(state, ClassishDefinitionType::Interface, vec![]);
}
let member_flags = self.interface_members_flags()?;
let member_flags = self.interface_members_flags(state)?;
peek_token!([
TokenKind::Const => self.parse_classish_const(member_flags),
TokenKind::Const => self.parse_classish_const(state, member_flags),
TokenKind::Function => self.method(
state,
ClassishDefinitionType::Interface,
member_flags.iter().map(|t| t.clone().into()).collect(),
)
], self, ["`const`", "`function`"])
], state, ["`const`", "`function`"])
}
pub(in crate::parser) fn trait_statement(&mut self) -> ParseResult<Statement> {
self.complete_class_statement(ClassishDefinitionType::Trait)
pub(in crate::parser) fn trait_statement(&self, state: &mut State) -> ParseResult<Statement> {
self.complete_class_statement(state, ClassishDefinitionType::Trait)
}
pub(in crate::parser) fn anonymous_class_statement(&mut self) -> ParseResult<Statement> {
self.complete_class_statement(ClassishDefinitionType::AnonymousClass)
pub(in crate::parser) fn anonymous_class_statement(
&self,
state: &mut State,
) -> ParseResult<Statement> {
self.complete_class_statement(state, ClassishDefinitionType::AnonymousClass)
}
pub(in crate::parser) fn enum_statement(&mut self, backed: bool) -> ParseResult<Statement> {
if self.current.kind == TokenKind::Case {
self.next();
pub(in crate::parser) fn enum_statement(
&self,
state: &mut State,
backed: bool,
) -> ParseResult<Statement> {
if state.current.kind == TokenKind::Case {
state.next();
let name = self.ident()?;
let name = self.ident(state)?;
if backed {
expect_token!([TokenKind::Equals], self, "`=`");
expect_token!([TokenKind::Equals], state, "`=`");
let value = self.expression(Precedence::Lowest)?;
self.semi()?;
let value = self.expression(state, Precedence::Lowest)?;
self.semi(state)?;
return Ok(Statement::BackedEnumCase {
name: name.into(),
value,
});
} else {
self.semi()?;
self.semi(state)?;
return Ok(Statement::UnitEnumCase { name: name.into() });
}
}
if self.current.kind == TokenKind::Const {
return self.parse_classish_const(vec![]);
if state.current.kind == TokenKind::Const {
return self.parse_classish_const(state, vec![]);
}
if self.current.kind == TokenKind::Function {
return self.method(ClassishDefinitionType::Enum, vec![]);
if state.current.kind == TokenKind::Function {
return self.method(state, ClassishDefinitionType::Enum, vec![]);
}
let member_flags = self.enum_members_flags()?;
let member_flags = self.enum_members_flags(state)?;
peek_token!([
TokenKind::Const => self.parse_classish_const(member_flags),
TokenKind::Const => self.parse_classish_const(state, member_flags),
TokenKind::Function => self.method(
state,
ClassishDefinitionType::Enum,
member_flags.iter().map(|t| t.clone().into()).collect(),
)
], self, ["`const`", "`function`"])
], state, ["`const`", "`function`"])
}
fn complete_class_statement(
&mut self,
&self,
state: &mut State,
class_type: ClassishDefinitionType,
) -> ParseResult<Statement> {
if self.current.kind == TokenKind::Use {
return self.parse_classish_uses();
if state.current.kind == TokenKind::Use {
return self.parse_classish_uses(state);
}
if self.current.kind == TokenKind::Var {
return self.parse_classish_var();
if state.current.kind == TokenKind::Var {
return self.parse_classish_var(state);
}
if self.current.kind == TokenKind::Const {
return self.parse_classish_const(vec![]);
if state.current.kind == TokenKind::Const {
return self.parse_classish_const(state, vec![]);
}
if self.current.kind == TokenKind::Function {
return self.method(class_type, vec![]);
if state.current.kind == TokenKind::Function {
return self.method(state, class_type, vec![]);
}
let member_flags = self.class_members_flags()?;
let member_flags = self.class_members_flags(state)?;
match &self.current.kind {
TokenKind::Const => self.parse_classish_const(member_flags),
match &state.current.kind {
TokenKind::Const => self.parse_classish_const(state, member_flags),
TokenKind::Function => self.method(
state,
class_type,
member_flags.iter().map(|t| t.clone().into()).collect(),
),
// TODO
TokenKind::Variable(_) => {
let var = self.var()?;
let var = self.var(state)?;
let mut value = None;
if self.current.kind == TokenKind::Equals {
self.next();
value = Some(self.expression(Precedence::Lowest)?);
if state.current.kind == TokenKind::Equals {
state.next();
value = Some(self.expression(state, Precedence::Lowest)?);
}
self.semi()?;
self.semi(state)?;
Ok(Statement::Property {
var,
@ -153,19 +169,19 @@ impl Parser {
| TokenKind::FullyQualifiedIdentifier(_)
| TokenKind::Array
| TokenKind::Null => {
let prop_type = self.type_string()?;
let var = self.var()?;
let prop_type = self.type_string(state)?;
let var = self.var(state)?;
let mut value = None;
if self.current.kind == TokenKind::Equals {
self.next();
value = Some(self.expression(Precedence::Lowest)?);
if state.current.kind == TokenKind::Equals {
state.next();
value = Some(self.expression(state, Precedence::Lowest)?);
}
// TODO: Support comma-separated property declarations.
// nikic/php-parser does this with a single Property statement
// that is capable of holding multiple property declarations.
self.semi()?;
self.semi(state)?;
Ok(Statement::Property {
var,
@ -176,30 +192,30 @@ impl Parser {
}
_ => expected_token_err!(
["`const`", "`function`", "an identifier", "a varaible"],
self
state
),
}
}
fn parse_classish_var(&mut self) -> ParseResult<Statement> {
self.next();
fn parse_classish_var(&self, state: &mut State) -> ParseResult<Statement> {
state.next();
let mut var_type = None;
if !matches!(self.current.kind, TokenKind::Variable(_)) || self.config.force_type_strings {
var_type = Some(self.type_string()?);
if !matches!(state.current.kind, TokenKind::Variable(_)) {
var_type = Some(self.type_string(state)?);
}
let var = self.var()?;
let var = self.var(state)?;
let mut value = None;
if self.current.kind == TokenKind::Equals {
self.next();
if state.current.kind == TokenKind::Equals {
state.next();
value = Some(self.expression(Precedence::Lowest)?);
value = Some(self.expression(state, Precedence::Lowest)?);
}
self.semi()?;
self.semi(state)?;
Ok(Statement::Var {
var,
@ -208,51 +224,52 @@ impl Parser {
})
}
fn parse_classish_uses(&mut self) -> ParseResult<Statement> {
self.next();
fn parse_classish_uses(&self, state: &mut State) -> ParseResult<Statement> {
state.next();
let mut traits = Vec::new();
while self.current.kind != TokenKind::SemiColon && self.current.kind != TokenKind::LeftBrace
while state.current.kind != TokenKind::SemiColon
&& state.current.kind != TokenKind::LeftBrace
{
self.optional_comma()?;
self.optional_comma(state)?;
let t = self.full_name()?;
let t = self.full_name(state)?;
traits.push(t.into());
}
let mut adaptations = Vec::new();
if self.current.kind == TokenKind::LeftBrace {
self.lbrace()?;
if state.current.kind == TokenKind::LeftBrace {
self.lbrace(state)?;
while self.current.kind != TokenKind::RightBrace {
let (r#trait, method): (Option<Identifier>, Identifier) = match self.peek.kind {
while state.current.kind != TokenKind::RightBrace {
let (r#trait, method): (Option<Identifier>, Identifier) = match state.peek.kind {
TokenKind::DoubleColon => {
let r#trait = self.full_name()?;
self.next();
let method = self.ident()?;
let r#trait = self.full_name(state)?;
state.next();
let method = self.ident(state)?;
(Some(r#trait.into()), method.into())
}
_ => (None, self.ident()?.into()),
_ => (None, self.ident(state)?.into()),
};
match self.current.kind {
match state.current.kind {
TokenKind::As => {
self.next();
state.next();
match self.current.kind {
match state.current.kind {
TokenKind::Public | TokenKind::Protected | TokenKind::Private => {
let visibility: MethodFlag = self.current.kind.clone().into();
self.next();
let visibility: MethodFlag = state.current.kind.clone().into();
state.next();
if self.current.kind == TokenKind::SemiColon {
if state.current.kind == TokenKind::SemiColon {
adaptations.push(TraitAdaptation::Visibility {
r#trait,
method,
visibility,
});
} else {
let alias: Identifier = self.name()?.into();
let alias: Identifier = self.name(state)?.into();
adaptations.push(TraitAdaptation::Alias {
r#trait,
method,
@ -262,7 +279,7 @@ impl Parser {
}
}
_ => {
let alias: Identifier = self.name()?.into();
let alias: Identifier = self.name(state)?.into();
adaptations.push(TraitAdaptation::Alias {
r#trait,
method,
@ -273,13 +290,13 @@ impl Parser {
}
}
TokenKind::Insteadof => {
self.next();
state.next();
let mut insteadof = Vec::new();
insteadof.push(self.full_name()?.into());
while self.current.kind != TokenKind::SemiColon {
self.optional_comma()?;
insteadof.push(self.full_name()?.into());
insteadof.push(self.full_name(state)?.into());
while state.current.kind != TokenKind::SemiColon {
self.optional_comma(state)?;
insteadof.push(self.full_name(state)?.into());
}
adaptations.push(TraitAdaptation::Precedence {
@ -290,18 +307,18 @@ impl Parser {
}
_ => {
return Err(ParseError::UnexpectedToken(
self.current.kind.to_string(),
self.current.span,
state.current.kind.to_string(),
state.current.span,
))
}
};
self.semi()?;
self.semi(state)?;
}
self.rbrace()?;
self.rbrace(state)?;
} else {
self.semi()?;
self.semi(state)?;
}
Ok(Statement::TraitUse {
@ -310,30 +327,34 @@ impl Parser {
})
}
fn parse_classish_const(&mut self, const_flags: Vec<TokenKind>) -> ParseResult<Statement> {
fn parse_classish_const(
&self,
state: &mut State,
const_flags: Vec<TokenKind>,
) -> ParseResult<Statement> {
if const_flags.contains(&TokenKind::Static) {
return Err(ParseError::StaticModifierOnConstant(self.current.span));
return Err(ParseError::StaticModifierOnConstant(state.current.span));
}
if const_flags.contains(&TokenKind::Readonly) {
return Err(ParseError::ReadonlyModifierOnConstant(self.current.span));
return Err(ParseError::ReadonlyModifierOnConstant(state.current.span));
}
if const_flags.contains(&TokenKind::Final) && const_flags.contains(&TokenKind::Private) {
return Err(ParseError::FinalModifierOnPrivateConstant(
self.current.span,
state.current.span,
));
}
self.next();
state.next();
let name = self.ident()?;
let name = self.ident(state)?;
expect_token!([TokenKind::Equals], self, "`=`");
expect_token!([TokenKind::Equals], state, "`=`");
let value = self.expression(Precedence::Lowest)?;
let value = self.expression(state, Precedence::Lowest)?;
self.semi()?;
self.semi(state)?;
Ok(Statement::ClassishConstant {
name: name.into(),

30
src/parser/comments.rs
View File

@ -1,30 +0,0 @@
use crate::lexer::token::Token;
use crate::lexer::token::TokenKind;
use crate::parser::Parser;
impl Parser {
pub(in crate::parser) fn skip_comments(&mut self) {
while matches!(
self.current.kind,
TokenKind::Comment(_) | TokenKind::DocComment(_)
) {
self.next();
}
}
pub(in crate::parser) fn gather_comments(&mut self) {
while matches!(
self.current.kind,
TokenKind::Comment(_) | TokenKind::DocComment(_)
) {
self.comments.push(self.current.clone());
self.next();
}
}
pub(in crate::parser) fn clear_comments(&mut self) -> Vec<Token> {
let c = self.comments.clone();
self.comments = vec![];
c
}
}

61
src/parser/flags.rs
View File

@ -1,6 +1,7 @@
use crate::lexer::token::TokenKind;
use crate::parser::error::ParseError;
use crate::parser::error::ParseResult;
use crate::parser::state::State;
use crate::parser::Parser;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
@ -13,22 +14,31 @@ enum FlagTarget {
}
impl Parser {
pub(in crate::parser) fn class_flags(&mut self) -> ParseResult<Vec<TokenKind>> {
pub(in crate::parser) fn class_flags(&self, state: &mut State) -> ParseResult<Vec<TokenKind>> {
self.collect(
state,
vec![TokenKind::Final, TokenKind::Abstract, TokenKind::Readonly],
FlagTarget::Class,
)
}
pub(in crate::parser) fn interface_members_flags(&mut self) -> ParseResult<Vec<TokenKind>> {
pub(in crate::parser) fn interface_members_flags(
&self,
state: &mut State,
) -> ParseResult<Vec<TokenKind>> {
self.collect(
state,
vec![TokenKind::Public, TokenKind::Static],
FlagTarget::InterfaceMember,
)
}
pub(in crate::parser) fn class_members_flags(&mut self) -> ParseResult<Vec<TokenKind>> {
pub(in crate::parser) fn class_members_flags(
&self,
state: &mut State,
) -> ParseResult<Vec<TokenKind>> {
self.collect(
state,
vec![
TokenKind::Final,
TokenKind::Abstract,
@ -42,8 +52,12 @@ impl Parser {
)
}
pub(in crate::parser) fn enum_members_flags(&mut self) -> ParseResult<Vec<TokenKind>> {
pub(in crate::parser) fn enum_members_flags(
&self,
state: &mut State,
) -> ParseResult<Vec<TokenKind>> {
self.collect(
state,
vec![
TokenKind::Final,
TokenKind::Private,
@ -55,8 +69,12 @@ impl Parser {
)
}
pub(in crate::parser) fn promoted_property_flags(&mut self) -> ParseResult<Vec<TokenKind>> {
pub(in crate::parser) fn promoted_property_flags(
&self,
state: &mut State,
) -> ParseResult<Vec<TokenKind>> {
self.collect(
state,
vec![
TokenKind::Private,
TokenKind::Protected,
@ -68,63 +86,64 @@ impl Parser {
}
fn collect(
&mut self,
&self,
state: &mut State,
flags: Vec<TokenKind>,
target: FlagTarget,
) -> ParseResult<Vec<TokenKind>> {
let mut collected: Vec<TokenKind> = vec![];
loop {
if flags.contains(&self.current.kind) {
if collected.contains(&self.current.kind) {
if flags.contains(&state.current.kind) {
if collected.contains(&state.current.kind) {
return Err(ParseError::MultipleModifiers(
self.current.kind.to_string(),
self.current.span,
state.current.kind.to_string(),
state.current.span,
));
}
match self.current.kind {
match state.current.kind {
TokenKind::Private
if collected.contains(&TokenKind::Protected)
|| collected.contains(&TokenKind::Public) =>
{
return Err(ParseError::MultipleAccessModifiers(self.current.span));
return Err(ParseError::MultipleAccessModifiers(state.current.span));
}
TokenKind::Protected
if collected.contains(&TokenKind::Private)
|| collected.contains(&TokenKind::Public) =>
{
return Err(ParseError::MultipleAccessModifiers(self.current.span));
return Err(ParseError::MultipleAccessModifiers(state.current.span));
}
TokenKind::Public
if collected.contains(&TokenKind::Private)
|| collected.contains(&TokenKind::Protected) =>
{
return Err(ParseError::MultipleAccessModifiers(self.current.span));
return Err(ParseError::MultipleAccessModifiers(state.current.span));
}
_ => {}
};
if matches!(target, FlagTarget::ClassMember | FlagTarget::Class) {
match self.current.kind {
match state.current.kind {
TokenKind::Final if collected.contains(&TokenKind::Abstract) => {
if target == FlagTarget::Class {
return Err(ParseError::FinalModifierOnAbstractClass(
self.current.span,
state.current.span,
));
} else {
return Err(ParseError::FinalModifierOnAbstractClassMember(
self.current.span,
state.current.span,
));
}
}
TokenKind::Abstract if collected.contains(&TokenKind::Final) => {
if target == FlagTarget::Class {
return Err(ParseError::FinalModifierOnAbstractClass(
self.current.span,
state.current.span,
));
} else {
return Err(ParseError::FinalModifierOnAbstractClassMember(
self.current.span,
state.current.span,
));
}
}
@ -132,8 +151,8 @@ impl Parser {
};
}
collected.push(self.current.kind.clone());
self.next();
collected.push(state.current.kind.clone());
state.next();
} else {
break;
}

62
src/parser/functions.rs
View File

@ -7,40 +7,41 @@ use crate::parser::classish_statement::ClassishDefinitionType;
use crate::parser::error::ParseError;
use crate::parser::error::ParseResult;
use crate::parser::params::ParamPosition;
use crate::parser::state::State;
use crate::parser::Parser;
impl Parser {
pub(in crate::parser) fn function(&mut self) -> ParseResult<Statement> {
self.next();
pub(in crate::parser) fn function(&self, state: &mut State) -> ParseResult<Statement> {
state.next();
let by_ref = if self.current.kind == TokenKind::Ampersand {
self.next();
let by_ref = if state.current.kind == TokenKind::Ampersand {
state.next();
true
} else {
false
};
let name = self.ident()?;
let name = self.ident(state)?;
self.lparen()?;
self.lparen(state)?;
let params = self.param_list(ParamPosition::Function)?;
let params = self.param_list(state, ParamPosition::Function)?;
self.rparen()?;
self.rparen(state)?;
let mut return_type = None;
if self.current.kind == TokenKind::Colon || self.config.force_type_strings {
self.colon()?;
if state.current.kind == TokenKind::Colon {
self.colon(state)?;
return_type = Some(self.type_string()?);
return_type = Some(self.type_string(state)?);
}
self.lbrace()?;
self.lbrace(state)?;
let body = self.block(&TokenKind::RightBrace)?;
let body = self.block(state, &TokenKind::RightBrace)?;
self.rbrace()?;
self.rbrace(state)?;
Ok(Statement::Function {
name: name.into(),
@ -52,7 +53,8 @@ impl Parser {
}
pub(in crate::parser) fn method(
&mut self,
&self,
state: &mut State,
class_type: ClassishDefinitionType,
flags: Vec<MethodFlag>,
) -> ParseResult<Statement> {
@ -62,13 +64,13 @@ impl Parser {
if !cf.contains(&ClassFlag::Abstract) && flags.contains(&MethodFlag::Abstract) =>
{
return Err(ParseError::AbstractModifierOnNonAbstractClassMethod(
self.current.span,
state.current.span,
));
}
_ => (),
}
self.next();
state.next();
let has_body = match &class_type {
ClassishDefinitionType::Class(_) | ClassishDefinitionType::Trait => {
@ -78,33 +80,33 @@ impl Parser {
ClassishDefinitionType::Enum | ClassishDefinitionType::AnonymousClass => true,
};
let by_ref = if self.current.kind == TokenKind::Ampersand {
self.next();
let by_ref = if state.current.kind == TokenKind::Ampersand {
state.next();
true
} else {
false
};
let name = self.ident_maybe_reserved()?;
let name = self.ident_maybe_reserved(state)?;
self.lparen()?;
self.lparen(state)?;
let position = position_from_flags_and_name(class_type, flags.clone(), name.clone());
let params = self.param_list(position)?;
let params = self.param_list(state, position)?;
self.rparen()?;
self.rparen(state)?;
let mut return_type = None;
if self.current.kind == TokenKind::Colon || self.config.force_type_strings {
self.colon()?;
if state.current.kind == TokenKind::Colon {
self.colon(state)?;
return_type = Some(self.type_string()?);
return_type = Some(self.type_string(state)?);
}
if !has_body {
self.semi()?;
self.semi(state)?;
Ok(Statement::AbstractMethod {
name: name.into(),
@ -114,11 +116,11 @@ impl Parser {
by_ref,
})
} else {
self.lbrace()?;
self.lbrace(state)?;
let body = self.block(&TokenKind::RightBrace)?;
let body = self.block(state, &TokenKind::RightBrace)?;
self.rbrace()?;
self.rbrace(state)?;
Ok(Statement::Method {
name: name.into(),

55
src/parser/ident.rs
View File

@ -1,71 +1,78 @@
use crate::lexer::byte_string::ByteString;
use crate::lexer::token::TokenKind;
use crate::parser::error::ParseResult;
use crate::parser::state::State;
use crate::parser::Parser;
use crate::expect_token;
impl Parser {
/// Expect an unqualified identifier such as Foo or Bar.
pub(in crate::parser) fn ident(&mut self) -> ParseResult<ByteString> {
pub(in crate::parser) fn ident(&self, state: &mut State) -> ParseResult<ByteString> {
Ok(expect_token!([
TokenKind::Identifier(identifier) => identifier,
], self, "an identifier"))
], state, "an identifier"))
}
/// Expect an unqualified or qualified identifier such as Foo, Bar or Foo\Bar.
pub(in crate::parser) fn name(&mut self) -> ParseResult<ByteString> {
pub(in crate::parser) fn name(&self, state: &mut State) -> ParseResult<ByteString> {
Ok(expect_token!([
TokenKind::Identifier(identifier) => identifier,
TokenKind::QualifiedIdentifier(qualified) => qualified,
], self, "an identifier"))
], state, "an identifier"))
}
/// Expect an unqualified, qualified or fully qualified identifier such as Foo, Foo\Bar or \Foo\Bar.
pub(in crate::parser) fn full_name(&mut self) -> ParseResult<ByteString> {
pub(in crate::parser) fn full_name(&self, state: &mut State) -> ParseResult<ByteString> {
Ok(expect_token!([
TokenKind::Identifier(identifier) => identifier,
TokenKind::QualifiedIdentifier(qualified) => qualified,
TokenKind::FullyQualifiedIdentifier(fully_qualified) => fully_qualified,
], self, "an identifier"))
], state, "an identifier"))
}
pub(in crate::parser) fn var(&mut self) -> ParseResult<ByteString> {
pub(in crate::parser) fn var(&self, state: &mut State) -> ParseResult<ByteString> {
Ok(expect_token!([
TokenKind::Variable(v) => v,
], self, "a variable"))
], state, "a variable"))
}
pub(in crate::parser) fn full_name_maybe_type_keyword(&mut self) -> ParseResult<ByteString> {
match self.current.kind {
pub(in crate::parser) fn full_name_maybe_type_keyword(
&self,
state: &mut State,
) -> ParseResult<ByteString> {
match state.current.kind {
TokenKind::Array | TokenKind::Callable => {
let r = Ok(self.current.kind.to_string().into());
self.next();
let r = Ok(state.current.kind.to_string().into());
state.next();
r
}
_ => self.full_name(),
_ => self.full_name(state),
}
}
pub(in crate::parser) fn type_with_static(&mut self) -> ParseResult<ByteString> {
Ok(match self.current.kind {
pub(in crate::parser) fn type_with_static(&self, state: &mut State) -> ParseResult<ByteString> {
Ok(match state.current.kind {
TokenKind::Static | TokenKind::Null | TokenKind::True | TokenKind::False => {
let str = self.current.kind.to_string();
self.next();
let str = state.current.kind.to_string();
state.next();
str.into()
}
_ => self.full_name_maybe_type_keyword()?,
_ => self.full_name_maybe_type_keyword(state)?,
})
}
pub(in crate::parser) fn ident_maybe_reserved(&mut self) -> ParseResult<ByteString> {
match self.current.kind {
_ if is_reserved_ident(&self.current.kind) => {
let string = self.current.kind.to_string().into();
self.next();
pub(in crate::parser) fn ident_maybe_reserved(
&self,
state: &mut State,
) -> ParseResult<ByteString> {
match state.current.kind {
_ if is_reserved_ident(&state.current.kind) => {
let string = state.current.kind.to_string().into();
state.next();
Ok(string)
}
_ => self.ident(),
_ => self.ident(state),
}
}
}

68
src/parser/macros.rs
View File

@ -1,68 +1,68 @@
#[macro_export]
macro_rules! peek_token {
([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {{
$parser.skip_comments();
match $parser.current.kind.clone() {
([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{
$state.skip_comments();
match $state.current.kind.clone() {
$(
$expected => $out,
)+
_ => {
return $crate::expected_token_err!([ $($message,)+ ], $parser);
return $crate::expected_token_err!([ $($message,)+ ], $state);
}
}
}};
([ $($expected:pat),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {{
$parser.skip_comments();
if !matches!($parser.current.kind, $(| $expected )+) {
return $crate::expected_token_err!([ $($message,)+ ], $parser);
([ $($expected:pat),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{
$state.skip_comments();
if !matches!($state.current.kind, $(| $expected )+) {
return $crate::expected_token_err!([ $($message,)+ ], $state);
}
}};
([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, $message:literal) => {
$crate::peek_token!([ $($expected => $out,)+ ], $parser, [$message])
([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, $message:literal) => {
$crate::peek_token!([ $($expected => $out,)+ ], $state, [$message])
};
([ $($expected:pat),+ $(,)? ], $parser:expr, $message:literal) => {
$crate::peek_token!([ $($expected,)+ ], $parser, [$message])
([ $($expected:pat),+ $(,)? ], $state:expr, $message:literal) => {
$crate::peek_token!([ $($expected,)+ ], $state, [$message])
};
}
#[macro_export]
macro_rules! expect_token {
([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {
$crate::peek_token!([ $($expected => { $parser.next(); $out },)+ ], $parser, [$($message,)+])
([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {
$crate::peek_token!([ $($expected => { $state.next(); $out },)+ ], $state, [$($message,)+])
};
([ $($expected:pat),+ $(,)? ], $parser:expr, [ $($message:literal),+ $(,)? ]) => {
$crate::peek_token!([ $($expected => { $parser.next(); },)+ ], $parser, [$($message,)+])
([ $($expected:pat),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {
$crate::peek_token!([ $($expected => { $state.next(); },)+ ], $state, [$($message,)+])
};
([ $($expected:pat => $out:expr),+ $(,)? ], $parser:expr, $message:literal) => {
$crate::peek_token!([ $($expected => { $parser.next(); $out },)+ ], $parser, [$message])
([ $($expected:pat => $out:expr),+ $(,)? ], $state:expr, $message:literal) => {
$crate::peek_token!([ $($expected => { $state.next(); $out },)+ ], $state, [$message])
};
([ $($expected:pat),+ $(,)? ], $parser:expr, $message:literal) => {
$crate::peek_token!([ $($expected => { $parser.next(); },)+ ], $parser, [$message])
([ $($expected:pat),+ $(,)? ], $state:expr, $message:literal) => {
$crate::peek_token!([ $($expected => { $state.next(); },)+ ], $state, [$message])
};
}
#[macro_export]
macro_rules! expect_literal {
($parser:expr) => {{
$parser.skip_comments();
match $parser.current.kind.clone() {
($state:expr) => {{
$state.skip_comments();
match $state.current.kind.clone() {
TokenKind::LiteralInteger(i) => {
let e = Expression::LiteralInteger { i };
$parser.next();
$state.next();
e
}
TokenKind::LiteralFloat(f) => {
let e = Expression::LiteralFloat { f };
$parser.next();
$state.next();
e
}
TokenKind::LiteralString(s) => {
let e = Expression::LiteralString { value: s.clone() };
$parser.next();
$state.next();
e
}
_ => {
return $crate::expected_token_err!(["a literal"], $parser);
return $crate::expected_token_err!(["a literal"], $state);
}
}
}};
@ -70,26 +70,26 @@ macro_rules! expect_literal {
#[macro_export]
macro_rules! expected_token_err {
([ $($expected:literal),+ $(,)? ], $parser:expr $(,)?) => {{
match &$parser.current.kind {
([ $($expected:literal),+ $(,)? ], $state:expr $(,)?) => {{
match &$state.current.kind {
TokenKind::Eof => {
Err($crate::parser::error::ParseError::ExpectedToken(
vec![$($expected.into()),+],
None,
$parser.current.span,
$state.current.span,
))
},
_ => {
Err($crate::parser::error::ParseError::ExpectedToken(
vec![$($expected.into()),+],
Some($parser.current.kind.to_string()),
$parser.current.span,
Some($state.current.kind.to_string()),
$state.current.span,
))
}
}
}};
($expected:literal, $parser:expr $(,)?) => {
$crate::expected_token_err!([$expected], $parser)
($expected:literal, $state:expr $(,)?) => {
$crate::expected_token_err!([$expected], $state)
};
}

1132
src/parser/mod.rs
View File

File diff suppressed because it is too large Load Diff

63
src/parser/params.rs
View File

@ -7,6 +7,7 @@ use crate::parser::ast::PropertyFlag;
use crate::parser::error::ParseError;
use crate::parser::error::ParseResult;
use crate::parser::precedence::Precedence;
use crate::parser::state::State;
use crate::parser::Parser;
use crate::expect_token;
@ -20,16 +21,17 @@ pub enum ParamPosition {
impl Parser {
pub(in crate::parser) fn param_list(
&mut self,
&self,
state: &mut State,
position: ParamPosition,
) -> Result<ParamList, ParseError> {
let mut params = ParamList::new();
while !self.is_eof() && self.current.kind != TokenKind::RightParen {
while !state.is_eof() && state.current.kind != TokenKind::RightParen {
let mut param_type = None;
let flags: Vec<PropertyFlag> = self
.promoted_property_flags()?
.promoted_property_flags(state)?
.iter()
.map(|f| f.into())
.collect();
@ -38,17 +40,17 @@ impl Parser {
match position {
ParamPosition::Method(name) if name != "__construct" => {
return Err(ParseError::PromotedPropertyOutsideConstructor(
self.current.span,
state.current.span,
));
}
ParamPosition::AbstractMethod(name) => {
if name == "__construct" {
return Err(ParseError::PromotedPropertyOnAbstractConstructor(
self.current.span,
state.current.span,
));
} else {
return Err(ParseError::PromotedPropertyOutsideConstructor(
self.current.span,
state.current.span,
));
}
}
@ -57,29 +59,28 @@ impl Parser {
}
// If this is a readonly promoted property, or we don't see a variable
if self.config.force_type_strings
|| flags.contains(&PropertyFlag::Readonly)
if flags.contains(&PropertyFlag::Readonly)
|| !matches!(
self.current.kind,
state.current.kind,
TokenKind::Variable(_) | TokenKind::Ellipsis | TokenKind::Ampersand
)
{
// Try to parse the type.
param_type = Some(self.type_string()?);
param_type = Some(self.type_string(state)?);
}
let mut variadic = false;
let mut by_ref = false;
if matches!(self.current.kind, TokenKind::Ampersand) {
self.next();
if matches!(state.current.kind, TokenKind::Ampersand) {
state.next();
by_ref = true;
}
if matches!(self.current.kind, TokenKind::Ellipsis) {
self.next();
if matches!(state.current.kind, TokenKind::Ellipsis) {
state.next();
if !flags.is_empty() {
return Err(ParseError::VariadicPromotedProperty(self.current.span));
return Err(ParseError::VariadicPromotedProperty(state.current.span));
}
variadic = true;
@ -88,12 +89,12 @@ impl Parser {
// 2. Then expect a variable.
let var = expect_token!([
TokenKind::Variable(v) => v
], self, "a varaible");
], state, "a varaible");
let mut default = None;
if self.current.kind == TokenKind::Equals {
self.next();
default = Some(self.expression(Precedence::Lowest)?);
if state.current.kind == TokenKind::Equals {
state.next();
default = Some(self.expression(state, Precedence::Lowest)?);
}
params.push(Param {
@ -105,29 +106,29 @@ impl Parser {
by_ref,
});
self.optional_comma()?;
self.optional_comma(state)?;
}
Ok(params)
}
pub(in crate::parser) fn args_list(&mut self) -> ParseResult<Vec<Arg>> {
pub(in crate::parser) fn args_list(&self, state: &mut State) -> ParseResult<Vec<Arg>> {
let mut args = Vec::new();
while !self.is_eof() && self.current.kind != TokenKind::RightParen {
while !state.is_eof() && state.current.kind != TokenKind::RightParen {
let mut name = None;
let mut unpack = false;
if matches!(self.current.kind, TokenKind::Identifier(_))
&& self.peek.kind == TokenKind::Colon
if matches!(state.current.kind, TokenKind::Identifier(_))
&& state.peek.kind == TokenKind::Colon
{
name = Some(self.ident_maybe_reserved()?);
self.next();
} else if self.current.kind == TokenKind::Ellipsis {
self.next();
name = Some(self.ident_maybe_reserved(state)?);
state.next();
} else if state.current.kind == TokenKind::Ellipsis {
state.next();
unpack = true;
}
if unpack && self.current.kind == TokenKind::RightParen {
if unpack && state.current.kind == TokenKind::RightParen {
args.push(Arg {
name: None,
unpack: false,
@ -137,7 +138,7 @@ impl Parser {
break;
}
let value = self.expression(Precedence::Lowest)?;
let value = self.expression(state, Precedence::Lowest)?;
args.push(Arg {
name,
@ -145,7 +146,7 @@ impl Parser {
value,
});
self.optional_comma()?;
self.optional_comma(state)?;
}
Ok(args)

35
src/parser/punc.rs
View File

@ -1,50 +1,51 @@
use crate::lexer::token::TokenKind;
use crate::parser::error::ParseResult;
use crate::parser::state::State;
use crate::parser::Parser;
use crate::expect_token;
impl Parser {
pub(in crate::parser) fn semi(&mut self) -> ParseResult<()> {
expect_token!([TokenKind::SemiColon], self, "`;`");
pub(in crate::parser) fn semi(&self, state: &mut State) -> ParseResult<()> {
expect_token!([TokenKind::SemiColon], state, "`;`");
Ok(())
}
pub(in crate::parser) fn lbrace(&mut self) -> ParseResult<()> {
expect_token!([TokenKind::LeftBrace], self, "`{`");
pub(in crate::parser) fn lbrace(&self, state: &mut State) -> ParseResult<()> {
expect_token!([TokenKind::LeftBrace], state, "`{`");
Ok(())
}
pub(in crate::parser) fn rbrace(&mut self) -> ParseResult<()> {
expect_token!([TokenKind::RightBrace], self, "`}`");
pub(in crate::parser) fn rbrace(&self, state: &mut State) -> ParseResult<()> {
expect_token!([TokenKind::RightBrace], state, "`}`");
Ok(())
}
pub(in crate::parser) fn lparen(&mut self) -> ParseResult<()> {
expect_token!([TokenKind::LeftParen], self, "`(`");
pub(in crate::parser) fn lparen(&self, state: &mut State) -> ParseResult<()> {
expect_token!([TokenKind::LeftParen], state, "`(`");
Ok(())
}
pub(in crate::parser) fn rparen(&mut self) -> ParseResult<()> {
expect_token!([TokenKind::RightParen], self, "`)`");
pub(in crate::parser) fn rparen(&self, state: &mut State) -> ParseResult<()> {
expect_token!([TokenKind::RightParen], state, "`)`");
Ok(())
}
pub(in crate::parser) fn rbracket(&mut self) -> ParseResult<()> {
expect_token!([TokenKind::RightBracket], self, "`]`");
pub(in crate::parser) fn rbracket(&self, state: &mut State) -> ParseResult<()> {
expect_token!([TokenKind::RightBracket], state, "`]`");
Ok(())
}
pub(in crate::parser) fn optional_comma(&mut self) -> ParseResult<()> {
if self.current.kind == TokenKind::Comma {
expect_token!([TokenKind::Comma], self, "`,`");
pub(in crate::parser) fn optional_comma(&self, state: &mut State) -> ParseResult<()> {
if state.current.kind == TokenKind::Comma {
expect_token!([TokenKind::Comma], state, "`,`");
}
Ok(())
}
pub(in crate::parser) fn colon(&mut self) -> ParseResult<()> {
expect_token!([TokenKind::Colon], self, "`:`");
pub(in crate::parser) fn colon(&self, state: &mut State) -> ParseResult<()> {
expect_token!([TokenKind::Colon], state, "`:`");
Ok(())
}

59
src/parser/state.rs Normal file
View File

@ -0,0 +1,59 @@
use std::vec::IntoIter;
use crate::lexer::token::Token;
use crate::lexer::token::TokenKind;
#[derive(Debug, Clone)]
pub struct State {
pub current: Token,
pub peek: Token,
pub iter: IntoIter<Token>,
pub comments: Vec<Token>,
}
impl State {
pub fn new(tokens: Vec<Token>) -> Self {
let mut iter = tokens.into_iter();
Self {
current: iter.next().unwrap_or_default(),
peek: iter.next().unwrap_or_default(),
iter,
comments: vec![],
}
}
pub fn skip_comments(&mut self) {
while matches!(
self.current.kind,
TokenKind::Comment(_) | TokenKind::DocComment(_)
) {
self.next();
}
}
pub fn gather_comments(&mut self) {
while matches!(
self.current.kind,
TokenKind::Comment(_) | TokenKind::DocComment(_)
) {
self.comments.push(self.current.clone());
self.next();
}
}
pub fn clear_comments(&mut self) -> Vec<Token> {
let c = self.comments.clone();
self.comments = vec![];
c
}
pub fn is_eof(&mut self) -> bool {
self.current.kind == TokenKind::Eof
}
pub fn next(&mut self) {
self.current = self.peek.clone();
self.peek = self.iter.next().unwrap_or_default()
}
}

19
src/parser/vars.rs
View File

@ -3,19 +3,20 @@ use crate::parser::ast::Expression;
use crate::parser::error::ParseError;
use crate::parser::error::ParseResult;
use crate::parser::precedence::Precedence;
use crate::parser::state::State;
use crate::parser::Parser;
impl Parser {
pub(in crate::parser) fn dynamic_variable(&mut self) -> ParseResult<Expression> {
self.next();
pub(in crate::parser) fn dynamic_variable(&self, state: &mut State) -> ParseResult<Expression> {
state.next();
Ok(match &self.current.kind {
Ok(match &state.current.kind {
TokenKind::LeftBrace => {
self.next();
state.next();
let name = self.expression(Precedence::Lowest)?;
let name = self.expression(state, Precedence::Lowest)?;
self.rbrace()?;
self.rbrace(state)?;
Expression::DynamicVariable {
name: Box::new(name),
@ -24,7 +25,7 @@ impl Parser {
TokenKind::Variable(variable) => {
let variable = variable.clone();
self.next();
state.next();
Expression::DynamicVariable {
name: Box::new(Expression::Variable { name: variable }),
@ -32,8 +33,8 @@ impl Parser {
}
_ => {
return Err(ParseError::UnexpectedToken(
self.current.kind.to_string(),
self.current.span,
state.current.kind.to_string(),
state.current.span,
))
}
})

2
tests/third_party_tests.rs
View File

@ -76,7 +76,7 @@ fn test_file(name: &str, filename: PathBuf) {
Lexer::new()
.tokenize(code.as_bytes())
.map(|tokens| {
Parser::new(None)
Parser::new()
.parse(tokens)
.map(|_| {
println!("✅ successfully parsed file: `\"{}\"`.", name);