diff --git a/src/lexer/byte_string.rs b/src/lexer/byte_string.rs index 64f4427..a95e54a 100644 --- a/src/lexer/byte_string.rs +++ b/src/lexer/byte_string.rs @@ -54,6 +54,12 @@ impl PartialEq<&[u8; N]> for ByteString { } } +impl PartialEq<&[u8; N]> for &ByteString { + fn eq(&self, other: &&[u8; N]) -> bool { + &self.0 == other + } +} + impl From> for ByteString { fn from(bytes: Vec) -> Self { ByteString::new(bytes) diff --git a/src/parser/internal/classish.rs b/src/parser/internal/classish.rs index ae7796a..5fffdaa 100644 --- a/src/parser/internal/classish.rs +++ b/src/parser/internal/classish.rs @@ -18,7 +18,7 @@ impl Parser { pub(in crate::parser) fn class_definition(&self, state: &mut State) -> ParseResult { let modifiers = self.get_class_modifier_group(self.modifiers(state)?)?; - expect_token!([TokenKind::Class], state, ["`class`"]); + self.skip(state, TokenKind::Class)?; let name = self.ident(state)?; @@ -80,7 +80,8 @@ impl Parser { &self, state: &mut State, ) -> ParseResult { - expect_token!([TokenKind::Interface], state, ["`interface`"]); + self.skip(state, TokenKind::Interface)?; + let name = self.ident(state)?; scoped!(state, Scope::Interface(name.clone()), { @@ -121,7 +122,7 @@ impl Parser { } pub(in crate::parser) fn trait_definition(&self, state: &mut State) -> ParseResult { - expect_token!([TokenKind::Trait], state, ["`trait`"]); + self.skip(state, TokenKind::Trait)?; let name = self.ident(state)?; @@ -155,11 +156,11 @@ impl Parser { &self, state: &mut State, ) -> ParseResult { - expect_token!([TokenKind::New], state, ["`new`"]); + self.skip(state, TokenKind::New)?; self.gather_attributes(state)?; - expect_token!([TokenKind::Class], state, ["`class`"]); + self.skip(state, TokenKind::Class)?; let mut args = vec![]; @@ -218,7 +219,7 @@ impl Parser { pub(in crate::parser) fn enum_definition(&self, state: &mut State) -> ParseResult { let start = state.current.span; - expect_token!([TokenKind::Enum], state, ["`enum`"]); + self.skip(state, TokenKind::Enum)?; let name = self.ident(state)?; diff --git a/src/parser/internal/classish_statements.rs b/src/parser/internal/classish_statements.rs index 6a61e5b..6436a95 100644 --- a/src/parser/internal/classish_statements.rs +++ b/src/parser/internal/classish_statements.rs @@ -118,7 +118,7 @@ impl Parser { )); } - expect_token!([TokenKind::Equals], state, "`=`"); + self.skip(state, TokenKind::Equals)?; let value = self.expression(state, Precedence::Lowest)?; @@ -404,7 +404,7 @@ impl Parser { let name = self.ident(state)?; - expect_token!([TokenKind::Equals], state, "`=`"); + self.skip(state, TokenKind::Equals)?; let value = self.expression(state, Precedence::Lowest)?; diff --git a/src/parser/internal/functions.rs b/src/parser/internal/functions.rs index c91eb37..1578cab 100644 --- a/src/parser/internal/functions.rs +++ b/src/parser/internal/functions.rs @@ -1,4 +1,3 @@ -use crate::expect_token; use crate::expected_scope; use crate::lexer::token::Span; use crate::lexer::token::TokenKind; @@ -34,7 +33,7 @@ impl Parser { false }; - expect_token!([TokenKind::Function], state, ["`function`"]); + self.skip(state, TokenKind::Function)?; let by_ref = if state.current.kind == TokenKind::Ampersand { state.next(); @@ -124,7 +123,7 @@ impl Parser { false }; - expect_token!([TokenKind::Fn], state, ["`fn`"]); + self.skip(state, TokenKind::Fn)?; let by_ref = if state.current.kind == TokenKind::Ampersand { state.next(); @@ -143,7 +142,7 @@ impl Parser { return_type = Some(self.get_type(state)?); } - expect_token!([TokenKind::DoubleArrow], state, ["`=>`"]); + self.skip(state, TokenKind::DoubleArrow)?; let body = scoped!(state, Scope::ArrowFunction(is_static), { Box::new(self.expression(state, Precedence::Lowest)?) @@ -166,7 +165,7 @@ impl Parser { pub(in crate::parser) fn function(&self, state: &mut State) -> ParseResult { let start = state.current.span; - expect_token!([TokenKind::Function], state, ["`function`"]); + self.skip(state, TokenKind::Function)?; let by_ref = if state.current.kind == TokenKind::Ampersand { state.next(); @@ -231,7 +230,7 @@ impl Parser { modifiers: MethodModifierGroup, start: Span, ) -> ParseResult { - expect_token!([TokenKind::Function], state, ["`function`"]); + self.skip(state, TokenKind::Function)?; let by_ref = if state.current.kind == TokenKind::Ampersand { state.next(); diff --git a/src/parser/internal/identifiers.rs b/src/parser/internal/identifiers.rs index fca833e..2e72c5b 100644 --- a/src/parser/internal/identifiers.rs +++ b/src/parser/internal/identifiers.rs @@ -12,7 +12,7 @@ impl Parser { pub(in crate::parser) fn ident(&self, state: &mut State) -> ParseResult { let name = peek_token!([ TokenKind::Identifier(identifier) => { - identifier + identifier.clone() }, ], state, "an identifier"); @@ -26,11 +26,8 @@ impl Parser { /// Expect an unqualified or qualified identifier such as Foo, Bar or Foo\Bar. pub(in crate::parser) fn name(&self, state: &mut State) -> ParseResult { let name = peek_token!([ - TokenKind::Identifier(identifier) => { - identifier - }, - TokenKind::QualifiedIdentifier(qualified) => { - qualified + TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) => { + name.clone() }, ], state, "an identifier"); @@ -43,29 +40,29 @@ impl Parser { /// Expect an optional unqualified or qualified identifier such as Foo, Bar or Foo\Bar. pub(in crate::parser) fn optional_name(&self, state: &mut State) -> Option { - match state.current.kind.clone() { + let ident = match &state.current.kind { TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) => { - let start = state.current.span; - state.next(); - let end = state.current.span; - - Some(Identifier { start, name, end }) + Some(Identifier { + start: state.current.span, + name: name.clone(), + end: state.peek.span, + }) } _ => None, + }; + + if ident.is_some() { + state.next(); } + + ident } /// Expect an unqualified, qualified or fully qualified identifier such as Foo, Foo\Bar or \Foo\Bar. pub(in crate::parser) fn full_name(&self, state: &mut State) -> ParseResult { let name = peek_token!([ - TokenKind::Identifier(identifier) => { - identifier - }, - TokenKind::QualifiedIdentifier(qualified) => { - qualified - }, - TokenKind::FullyQualifiedIdentifier(fully_qualified) => { - fully_qualified + TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) | TokenKind::FullyQualifiedIdentifier(name) => { + name.clone() }, ], state, "an identifier"); @@ -78,7 +75,7 @@ impl Parser { pub(in crate::parser) fn var(&self, state: &mut State) -> ParseResult { let name = peek_token!([ - TokenKind::Variable(v) => v, + TokenKind::Variable(v) => v.clone(), ], state, "a variable"); let start = state.current.span; diff --git a/src/parser/internal/mod.rs b/src/parser/internal/mod.rs index fbf1429..2387cf7 100644 --- a/src/parser/internal/mod.rs +++ b/src/parser/internal/mod.rs @@ -9,6 +9,7 @@ pub mod namespaces; pub mod parameters; pub mod precedences; pub mod punctuations; +pub mod tokens; pub mod try_block; pub mod types; pub mod variables; diff --git a/src/parser/internal/punctuations.rs b/src/parser/internal/punctuations.rs index 1780ab7..5500fc1 100644 --- a/src/parser/internal/punctuations.rs +++ b/src/parser/internal/punctuations.rs @@ -32,152 +32,30 @@ impl Parser { } pub(in crate::parser) fn left_brace(&self, state: &mut State) -> ParseResult { - state.skip_comments(); - - let end = state.current.span; - - if state.current.kind == TokenKind::LeftBrace { - state.next(); - state.skip_comments(); - } else { - let found = if state.current.kind == TokenKind::Eof { - None - } else { - Some(state.current.kind.to_string()) - }; - - return Err(ParseError::ExpectedToken( - vec!["`{`".to_string()], - found, - state.current.span, - )); - } - - Ok(end) + self.skip(state, TokenKind::LeftBrace) } pub(in crate::parser) fn right_brace(&self, state: &mut State) -> ParseResult { - state.skip_comments(); - - let end = state.current.span; - - if state.current.kind == TokenKind::RightBrace { - state.next(); - state.skip_comments(); - } else { - let found = if state.current.kind == TokenKind::Eof { - None - } else { - Some(state.current.kind.to_string()) - }; - - return Err(ParseError::ExpectedToken( - vec!["`}`".to_string()], - found, - state.current.span, - )); - } - - Ok(end) + self.skip(state, TokenKind::RightBrace) } pub(in crate::parser) fn left_parenthesis(&self, state: &mut State) -> ParseResult { - state.skip_comments(); - - let end = state.current.span; - - if state.current.kind == TokenKind::LeftParen { - state.next(); - state.skip_comments(); - } else { - let found = if state.current.kind == TokenKind::Eof { - None - } else { - Some(state.current.kind.to_string()) - }; - - return Err(ParseError::ExpectedToken( - vec!["`(`".to_string()], - found, - state.current.span, - )); - } - - Ok(end) + self.skip(state, TokenKind::LeftParen) } pub(in crate::parser) fn right_parenthesis(&self, state: &mut State) -> ParseResult { - state.skip_comments(); - - let end = state.current.span; - - if state.current.kind == TokenKind::RightParen { - state.next(); - state.skip_comments(); - } else { - let found = if state.current.kind == TokenKind::Eof { - None - } else { - Some(state.current.kind.to_string()) - }; - - return Err(ParseError::ExpectedToken( - vec!["`)`".to_string()], - found, - state.current.span, - )); - } - - Ok(end) + self.skip(state, TokenKind::RightParen) } pub(in crate::parser) fn right_bracket(&self, state: &mut State) -> ParseResult { - state.skip_comments(); + self.skip(state, TokenKind::RightBracket) + } - let end = state.current.span; - - if state.current.kind == TokenKind::RightBracket { - state.next(); - state.skip_comments(); - } else { - let found = if state.current.kind == TokenKind::Eof { - None - } else { - Some(state.current.kind.to_string()) - }; - - return Err(ParseError::ExpectedToken( - vec!["`]`".to_string()], - found, - state.current.span, - )); - } - - Ok(end) + pub(in crate::parser) fn double_arrow(&self, state: &mut State) -> ParseResult { + self.skip(state, TokenKind::DoubleArrow) } pub(in crate::parser) fn colon(&self, state: &mut State) -> ParseResult { - state.skip_comments(); - - let end = state.current.span; - - if state.current.kind == TokenKind::Colon { - state.next(); - state.skip_comments(); - } else { - let found = if state.current.kind == TokenKind::Eof { - None - } else { - Some(state.current.kind.to_string()) - }; - - return Err(ParseError::ExpectedToken( - vec!["`:`".to_string()], - found, - state.current.span, - )); - } - - Ok(end) + self.skip(state, TokenKind::Colon) } } diff --git a/src/parser/internal/tokens.rs b/src/parser/internal/tokens.rs new file mode 100644 index 0000000..495829b --- /dev/null +++ b/src/parser/internal/tokens.rs @@ -0,0 +1,62 @@ +use crate::lexer::token::Span; +use crate::lexer::token::TokenKind; +use crate::parser::error::ParseError; +use crate::parser::error::ParseResult; +use crate::parser::state::State; +use crate::parser::Parser; + +impl Parser { + pub(in crate::parser) fn skip(&self, state: &mut State, kind: TokenKind) -> ParseResult { + state.skip_comments(); + + if state.current.kind == kind { + let end = state.current.span; + + state.next(); + state.skip_comments(); + + Ok(end) + } else { + let found = if state.current.kind == TokenKind::Eof { + None + } else { + Some(state.current.kind.to_string()) + }; + + Err(ParseError::ExpectedToken( + vec![format!("`{}`", kind)], + found, + state.current.span, + )) + } + } + + pub(in crate::parser) fn skip_any_of( + &self, + state: &mut State, + kinds: &[TokenKind], + ) -> ParseResult { + state.skip_comments(); + + if kinds.contains(&state.current.kind) { + let end = state.current.span; + + state.next(); + state.skip_comments(); + + Ok(end) + } else { + let found = if state.current.kind == TokenKind::Eof { + None + } else { + Some(state.current.kind.to_string()) + }; + + Err(ParseError::ExpectedToken( + kinds.iter().map(|kind| format!("`{}`", kind)).collect(), + found, + state.current.span, + )) + } + } +} diff --git a/src/parser/internal/types.rs b/src/parser/internal/types.rs index b840ca1..5ecb594 100644 --- a/src/parser/internal/types.rs +++ b/src/parser/internal/types.rs @@ -1,4 +1,3 @@ -use crate::expect_token; use crate::expected_token; use crate::lexer::token::TokenKind; use crate::parser::ast::identifiers::Identifier; @@ -236,7 +235,8 @@ impl Parser { let mut types = vec![other]; - expect_token!([TokenKind::Pipe], state, ["|"]); + self.skip(state, TokenKind::Pipe)?; + loop { let ty = if state.current.kind == TokenKind::LeftParen { if within_dnf { @@ -305,7 +305,8 @@ impl Parser { let mut types = vec![other]; - expect_token!([TokenKind::Ampersand], state, ["&"]); + self.skip(state, TokenKind::Ampersand)?; + loop { let ty = if state.current.kind == TokenKind::LeftParen { if within_dnf { diff --git a/src/parser/macros.rs b/src/parser/macros.rs index b5cb5f7..70ccefe 100644 --- a/src/parser/macros.rs +++ b/src/parser/macros.rs @@ -2,7 +2,7 @@ macro_rules! peek_token { ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ $state.skip_comments(); - match $state.current.kind.clone() { + match &$state.current.kind { $( $( $pattern )|+ $( if $guard )? => $out, )+ @@ -27,17 +27,33 @@ macro_rules! peek_token { #[macro_export] macro_rules! expect_token { - ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { - $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); $out },)+ ], $state, [$($message,)+]) - }; - ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => { - $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); },)+ ], $state, [$($message,)+]) - }; + ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{ + $state.skip_comments(); + let token = $state.pull(); + match token.kind { + $( + $( $pattern )|+ $( if $guard )? => { + $out + }, + )+ + TokenKind::Eof => { + return Err($crate::parser::error::ParseError::ExpectedToken( + vec![$($message.into(),)+], + None, + token.span, + )) + }, + _ => { + return Err($crate::parser::error::ParseError::ExpectedToken( + vec![$($message.into(),)+], + Some(token.kind.to_string()), + token.span, + )) + } + } + }}; ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, $message:literal) => { - $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); $out },)+ ], $state, [$message]) - }; - ([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, $message:literal) => { - $crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); },)+ ], $state, [$message]) + $crate::expect_token!([ $($( $pattern )|+ $( if $guard )? => $out,)+ ], $state, [$message]) }; } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 2336544..7c57441 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -143,7 +143,7 @@ impl Parser { loop { let name = self.ident(state)?; - expect_token!([TokenKind::Equals], state, "`=`"); + self.skip(state, TokenKind::Equals)?; let value = self.expression(state, Precedence::Lowest)?; @@ -297,7 +297,7 @@ impl Parser { loop { let key = self.ident(state)?; - expect_token!([TokenKind::Equals], state, "`=`"); + self.skip(state, TokenKind::Equals)?; let value = expect_literal!(state); @@ -320,7 +320,7 @@ impl Parser { } else if state.current.kind == TokenKind::Colon { self.colon(state)?; let b = self.block(state, &TokenKind::EndDeclare)?; - expect_token!([TokenKind::EndDeclare], state, "`enddeclare`"); + self.skip(state, TokenKind::EndDeclare)?; self.semicolon(state)?; b } else { @@ -446,7 +446,7 @@ impl Parser { let body = self.block(state, &TokenKind::RightBrace)?; self.right_brace(state)?; - expect_token!([TokenKind::While], state, "`while`"); + self.skip(state, TokenKind::While)?; self.left_parenthesis(state)?; let condition = self.expression(state, Precedence::Lowest)?; @@ -476,7 +476,7 @@ impl Parser { if end_token == TokenKind::RightBrace { self.right_brace(state)?; } else { - expect_token!([TokenKind::EndWhile], state, "`endwhile`"); + self.skip(state, TokenKind::EndWhile)?; self.semicolon(state)?; } @@ -548,7 +548,7 @@ impl Parser { let then = self.block(state, &end_token)?; if end_token == TokenKind::EndFor { - expect_token!([TokenKind::EndFor], state, "`endfor`"); + self.skip(state, TokenKind::EndFor)?; self.semicolon(state)?; } else { self.right_brace(state)?; @@ -568,7 +568,7 @@ impl Parser { let expr = self.expression(state, Precedence::Lowest)?; - expect_token!([TokenKind::As], state, ["`as`"]); + self.skip(state, TokenKind::As)?; let mut by_ref = state.current.kind == TokenKind::Ampersand; if by_ref { @@ -604,7 +604,7 @@ impl Parser { let body = self.block(state, &end_token)?; if end_token == TokenKind::EndForeach { - expect_token!([TokenKind::EndForeach], state, "`endforeach`"); + self.skip(state, TokenKind::EndForeach)?; self.semicolon(state)?; } else { self.right_brace(state)?; @@ -643,11 +643,8 @@ impl Parser { let condition = self.expression(state, Precedence::Lowest)?; - expect_token!( - [TokenKind::Colon, TokenKind::SemiColon], - state, - ["`:`", "`;`"] - ); + self.skip_any_of(state, &[TokenKind::Colon, TokenKind::SemiColon])?; + let mut body = Block::new(); while state.current.kind != TokenKind::Case @@ -666,11 +663,7 @@ impl Parser { TokenKind::Default => { state.next(); - expect_token!( - [TokenKind::Colon, TokenKind::SemiColon], - state, - ["`:`", "`;`"] - ); + self.skip_any_of(state, &[TokenKind::Colon, TokenKind::SemiColon])?; let mut body = Block::new(); @@ -693,7 +686,7 @@ impl Parser { } if end_token == TokenKind::EndSwitch { - expect_token!([TokenKind::EndSwitch], state, ["`endswitch`"]); + self.skip(state, TokenKind::EndSwitch)?; self.semicolon(state)?; } else { self.right_brace(state)?; @@ -760,7 +753,8 @@ impl Parser { r#else = Some(body); } - expect_token!([TokenKind::EndIf], state, ["`endif`"]); + self.skip(state, TokenKind::EndIf)?; + self.semicolon(state)?; Statement::If { @@ -817,7 +811,7 @@ impl Parser { }); } - expect_token!([TokenKind::Else], state, ["`else`"]); + self.skip(state, TokenKind::Else)?; self.left_brace(state)?; @@ -1182,7 +1176,7 @@ impl Parser { state.next(); } - expect_token!([TokenKind::DoubleArrow], state, "`=>`"); + self.double_arrow(state)?; let body = self.expression(state, Precedence::Lowest)?; @@ -1200,7 +1194,7 @@ impl Parser { } if !conditions.is_empty() { - expect_token!([TokenKind::DoubleArrow], state, "`=>`"); + self.double_arrow(state)?; } else { break; } @@ -1559,7 +1553,7 @@ impl Parser { } else { let index = self.expression(state, Precedence::Lowest)?; - expect_token!([TokenKind::RightBracket], state, ["`]`"]); + self.right_bracket(state)?; Expression::ArrayIndex { array: Box::new(lhs), @@ -1777,9 +1771,13 @@ impl Parser { // FIXME: This feels hacky. We should probably produce different tokens from the lexer // but since I already had the logic in place for parsing heredocs, this was // the fastest way to get nowdocs working too. - let mut s = - expect_token!([TokenKind::StringPart(s) => s], state, "constant string"); - let (indentation_type, indentation_amount) = expect_token!([TokenKind::EndDocString(_, indentation_type, indentation_amount) => (indentation_type, indentation_amount)], state, "label"); + let mut s = expect_token!([ + TokenKind::StringPart(s) => s + ], state, "constant string"); + + let (indentation_type, indentation_amount) = expect_token!([ + TokenKind::EndDocString(_, indentation_type, indentation_amount) => (indentation_type, indentation_amount) + ], state, "label"); // FIXME: Hacky code, but it's late and I want to get this done. if let Some(indentation_type) = indentation_type { @@ -1866,7 +1864,7 @@ impl Parser { // "{$expr}" state.next(); let e = self.expression(state, Precedence::Lowest)?; - expect_token!([TokenKind::RightBrace], state, "`}`"); + self.right_brace(state)?; Some(StringPart::Expr(Box::new(e))) } TokenKind::Variable(_) => { @@ -1912,7 +1910,8 @@ impl Parser { } }; - expect_token!([TokenKind::RightBracket], state, "`]`"); + self.right_bracket(state)?; + Expression::ArrayIndex { array: Box::new(var), index: Some(Box::new(index)), diff --git a/src/parser/state.rs b/src/parser/state.rs index 702cea7..dd67c9c 100644 --- a/src/parser/state.rs +++ b/src/parser/state.rs @@ -180,6 +180,20 @@ impl State { self.current.kind == TokenKind::Eof } + pub fn pull(&mut self) -> Token { + let mut current: Token = Default::default(); + + std::mem::swap(&mut current, &mut self.current); + std::mem::swap(&mut self.current, &mut self.peek); + + // peek already contains default. + if let Some(t) = self.iter.next() { + self.peek = t; + } + + current + } + pub fn next(&mut self) { // move peek to current std::mem::swap(&mut self.current, &mut self.peek);