danog/parser
1
0
Fork 0
mirror of https://github.com/danog/parser.git synced 2024-11-26 20:04:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

chore: avoid cloning tokens (#166)

Browse Source 
Signed-off-by: azjezz <azjezz@protonmail.com>
This commit is contained in:
Saif Eddin Gmati 2022-12-06 14:05:20 +01:00 committed by GitHub
parent c02b4db78d
commit 34b9f80e46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 183 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/lexer/byte_string.rs
View File

@ -54,6 +54,12 @@ impl<const N: usize> PartialEq<&[u8; N]> for ByteString {
}
}
impl<const N: usize> PartialEq<&[u8; N]> for &ByteString {
fn eq(&self, other: &&[u8; N]) -> bool {
&self.0 == other
}
}
impl From<Vec<u8>> for ByteString {
fn from(bytes: Vec<u8>) -> Self {
ByteString::new(bytes)

13
src/parser/internal/classish.rs
View File

@ -18,7 +18,7 @@ impl Parser {
pub(in crate::parser) fn class_definition(&self, state: &mut State) -> ParseResult<Statement> {
let modifiers = self.get_class_modifier_group(self.modifiers(state)?)?;
expect_token!([TokenKind::Class], state, ["`class`"]);
self.skip(state, TokenKind::Class)?;
let name = self.ident(state)?;
@ -80,7 +80,8 @@ impl Parser {
&self,
state: &mut State,
) -> ParseResult<Statement> {
expect_token!([TokenKind::Interface], state, ["`interface`"]);
self.skip(state, TokenKind::Interface)?;
let name = self.ident(state)?;
scoped!(state, Scope::Interface(name.clone()), {
@ -121,7 +122,7 @@ impl Parser {
}
pub(in crate::parser) fn trait_definition(&self, state: &mut State) -> ParseResult<Statement> {
expect_token!([TokenKind::Trait], state, ["`trait`"]);
self.skip(state, TokenKind::Trait)?;
let name = self.ident(state)?;
@ -155,11 +156,11 @@ impl Parser {
&self,
state: &mut State,
) -> ParseResult<Expression> {
expect_token!([TokenKind::New], state, ["`new`"]);
self.skip(state, TokenKind::New)?;
self.gather_attributes(state)?;
expect_token!([TokenKind::Class], state, ["`class`"]);
self.skip(state, TokenKind::Class)?;
let mut args = vec![];
@ -218,7 +219,7 @@ impl Parser {
pub(in crate::parser) fn enum_definition(&self, state: &mut State) -> ParseResult<Statement> {
let start = state.current.span;
expect_token!([TokenKind::Enum], state, ["`enum`"]);
self.skip(state, TokenKind::Enum)?;
let name = self.ident(state)?;

4
src/parser/internal/classish_statements.rs
View File

@ -118,7 +118,7 @@ impl Parser {
));
}
expect_token!([TokenKind::Equals], state, "`=`");
self.skip(state, TokenKind::Equals)?;
let value = self.expression(state, Precedence::Lowest)?;
@ -404,7 +404,7 @@ impl Parser {
let name = self.ident(state)?;
expect_token!([TokenKind::Equals], state, "`=`");
self.skip(state, TokenKind::Equals)?;
let value = self.expression(state, Precedence::Lowest)?;

11
src/parser/internal/functions.rs
View File

@ -1,4 +1,3 @@
use crate::expect_token;
use crate::expected_scope;
use crate::lexer::token::Span;
use crate::lexer::token::TokenKind;
@ -34,7 +33,7 @@ impl Parser {
false
};
expect_token!([TokenKind::Function], state, ["`function`"]);
self.skip(state, TokenKind::Function)?;
let by_ref = if state.current.kind == TokenKind::Ampersand {
state.next();
@ -124,7 +123,7 @@ impl Parser {
false
};
expect_token!([TokenKind::Fn], state, ["`fn`"]);
self.skip(state, TokenKind::Fn)?;
let by_ref = if state.current.kind == TokenKind::Ampersand {
state.next();
@ -143,7 +142,7 @@ impl Parser {
return_type = Some(self.get_type(state)?);
}
expect_token!([TokenKind::DoubleArrow], state, ["`=>`"]);
self.skip(state, TokenKind::DoubleArrow)?;
let body = scoped!(state, Scope::ArrowFunction(is_static), {
Box::new(self.expression(state, Precedence::Lowest)?)
@ -166,7 +165,7 @@ impl Parser {
pub(in crate::parser) fn function(&self, state: &mut State) -> ParseResult<Statement> {
let start = state.current.span;
expect_token!([TokenKind::Function], state, ["`function`"]);
self.skip(state, TokenKind::Function)?;
let by_ref = if state.current.kind == TokenKind::Ampersand {
state.next();
@ -231,7 +230,7 @@ impl Parser {
modifiers: MethodModifierGroup,
start: Span,
) -> ParseResult<Method> {
expect_token!([TokenKind::Function], state, ["`function`"]);
self.skip(state, TokenKind::Function)?;
let by_ref = if state.current.kind == TokenKind::Ampersand {
state.next();

39
src/parser/internal/identifiers.rs
View File

@ -12,7 +12,7 @@ impl Parser {
pub(in crate::parser) fn ident(&self, state: &mut State) -> ParseResult<Identifier> {
let name = peek_token!([
TokenKind::Identifier(identifier) => {
identifier
identifier.clone()
},
], state, "an identifier");
@ -26,11 +26,8 @@ impl Parser {
/// Expect an unqualified or qualified identifier such as Foo, Bar or Foo\Bar.
pub(in crate::parser) fn name(&self, state: &mut State) -> ParseResult<Identifier> {
let name = peek_token!([
TokenKind::Identifier(identifier) => {
identifier
},
TokenKind::QualifiedIdentifier(qualified) => {
qualified
TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) => {
name.clone()
},
], state, "an identifier");
@ -43,29 +40,29 @@ impl Parser {
/// Expect an optional unqualified or qualified identifier such as Foo, Bar or Foo\Bar.
pub(in crate::parser) fn optional_name(&self, state: &mut State) -> Option<Identifier> {
match state.current.kind.clone() {
let ident = match &state.current.kind {
TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) => {
let start = state.current.span;
state.next();
let end = state.current.span;
Some(Identifier { start, name, end })
Some(Identifier {
start: state.current.span,
name: name.clone(),
end: state.peek.span,
})
}
_ => None,
};
if ident.is_some() {
state.next();
}
ident
}
/// Expect an unqualified, qualified or fully qualified identifier such as Foo, Foo\Bar or \Foo\Bar.
pub(in crate::parser) fn full_name(&self, state: &mut State) -> ParseResult<Identifier> {
let name = peek_token!([
TokenKind::Identifier(identifier) => {
identifier
},
TokenKind::QualifiedIdentifier(qualified) => {
qualified
},
TokenKind::FullyQualifiedIdentifier(fully_qualified) => {
fully_qualified
TokenKind::Identifier(name) | TokenKind::QualifiedIdentifier(name) | TokenKind::FullyQualifiedIdentifier(name) => {
name.clone()
},
], state, "an identifier");
@ -78,7 +75,7 @@ impl Parser {
pub(in crate::parser) fn var(&self, state: &mut State) -> ParseResult<Variable> {
let name = peek_token!([
TokenKind::Variable(v) => v,
TokenKind::Variable(v) => v.clone(),
], state, "a variable");
let start = state.current.span;

1
src/parser/internal/mod.rs
View File

@ -9,6 +9,7 @@ pub mod namespaces;
pub mod parameters;
pub mod precedences;
pub mod punctuations;
pub mod tokens;
pub mod try_block;
pub mod types;
pub mod variables;

140
src/parser/internal/punctuations.rs
View File

@ -32,152 +32,30 @@ impl Parser {
}
pub(in crate::parser) fn left_brace(&self, state: &mut State) -> ParseResult<Span> {
state.skip_comments();
let end = state.current.span;
if state.current.kind == TokenKind::LeftBrace {
state.next();
state.skip_comments();
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
return Err(ParseError::ExpectedToken(
vec!["`{`".to_string()],
found,
state.current.span,
));
}
Ok(end)
self.skip(state, TokenKind::LeftBrace)
}
pub(in crate::parser) fn right_brace(&self, state: &mut State) -> ParseResult<Span> {
state.skip_comments();
let end = state.current.span;
if state.current.kind == TokenKind::RightBrace {
state.next();
state.skip_comments();
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
return Err(ParseError::ExpectedToken(
vec!["`}`".to_string()],
found,
state.current.span,
));
}
Ok(end)
self.skip(state, TokenKind::RightBrace)
}
pub(in crate::parser) fn left_parenthesis(&self, state: &mut State) -> ParseResult<Span> {
state.skip_comments();
let end = state.current.span;
if state.current.kind == TokenKind::LeftParen {
state.next();
state.skip_comments();
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
return Err(ParseError::ExpectedToken(
vec!["`(`".to_string()],
found,
state.current.span,
));
}
Ok(end)
self.skip(state, TokenKind::LeftParen)
}
pub(in crate::parser) fn right_parenthesis(&self, state: &mut State) -> ParseResult<Span> {
state.skip_comments();
let end = state.current.span;
if state.current.kind == TokenKind::RightParen {
state.next();
state.skip_comments();
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
return Err(ParseError::ExpectedToken(
vec!["`)`".to_string()],
found,
state.current.span,
));
}
Ok(end)
self.skip(state, TokenKind::RightParen)
}
pub(in crate::parser) fn right_bracket(&self, state: &mut State) -> ParseResult<Span> {
state.skip_comments();
self.skip(state, TokenKind::RightBracket)
}
let end = state.current.span;
if state.current.kind == TokenKind::RightBracket {
state.next();
state.skip_comments();
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
return Err(ParseError::ExpectedToken(
vec!["`]`".to_string()],
found,
state.current.span,
));
}
Ok(end)
pub(in crate::parser) fn double_arrow(&self, state: &mut State) -> ParseResult<Span> {
self.skip(state, TokenKind::DoubleArrow)
}
pub(in crate::parser) fn colon(&self, state: &mut State) -> ParseResult<Span> {
state.skip_comments();
let end = state.current.span;
if state.current.kind == TokenKind::Colon {
state.next();
state.skip_comments();
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
return Err(ParseError::ExpectedToken(
vec!["`:`".to_string()],
found,
state.current.span,
));
}
Ok(end)
self.skip(state, TokenKind::Colon)
}
}

62
src/parser/internal/tokens.rs Normal file
View File

@ -0,0 +1,62 @@
use crate::lexer::token::Span;
use crate::lexer::token::TokenKind;
use crate::parser::error::ParseError;
use crate::parser::error::ParseResult;
use crate::parser::state::State;
use crate::parser::Parser;
impl Parser {
pub(in crate::parser) fn skip(&self, state: &mut State, kind: TokenKind) -> ParseResult<Span> {
state.skip_comments();
if state.current.kind == kind {
let end = state.current.span;
state.next();
state.skip_comments();
Ok(end)
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
Err(ParseError::ExpectedToken(
vec![format!("`{}`", kind)],
found,
state.current.span,
))
}
}
pub(in crate::parser) fn skip_any_of(
&self,
state: &mut State,
kinds: &[TokenKind],
) -> ParseResult<Span> {
state.skip_comments();
if kinds.contains(&state.current.kind) {
let end = state.current.span;
state.next();
state.skip_comments();
Ok(end)
} else {
let found = if state.current.kind == TokenKind::Eof {
None
} else {
Some(state.current.kind.to_string())
};
Err(ParseError::ExpectedToken(
kinds.iter().map(|kind| format!("`{}`", kind)).collect(),
found,
state.current.span,
))
}
}
}

7
src/parser/internal/types.rs
View File

@ -1,4 +1,3 @@
use crate::expect_token;
use crate::expected_token;
use crate::lexer::token::TokenKind;
use crate::parser::ast::identifiers::Identifier;
@ -236,7 +235,8 @@ impl Parser {
let mut types = vec![other];
expect_token!([TokenKind::Pipe], state, ["|"]);
self.skip(state, TokenKind::Pipe)?;
loop {
let ty = if state.current.kind == TokenKind::LeftParen {
if within_dnf {
@ -305,7 +305,8 @@ impl Parser {
let mut types = vec![other];
expect_token!([TokenKind::Ampersand], state, ["&"]);
self.skip(state, TokenKind::Ampersand)?;
loop {
let ty = if state.current.kind == TokenKind::LeftParen {
if within_dnf {

38
src/parser/macros.rs
View File

@ -2,7 +2,7 @@
macro_rules! peek_token {
([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{
$state.skip_comments();
match $state.current.kind.clone() {
match &$state.current.kind {
$(
$( $pattern )|+ $( if $guard )? => $out,
)+
@ -27,17 +27,33 @@ macro_rules! peek_token {
#[macro_export]
macro_rules! expect_token {
([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {
$crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); $out },)+ ], $state, [$($message,)+])
};
([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {
$crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); },)+ ], $state, [$($message,)+])
};
([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, [ $($message:literal),+ $(,)? ]) => {{
$state.skip_comments();
let token = $state.pull();
match token.kind {
$(
$( $pattern )|+ $( if $guard )? => {
$out
},
)+
TokenKind::Eof => {
return Err($crate::parser::error::ParseError::ExpectedToken(
vec![$($message.into(),)+],
None,
token.span,
))
},
_ => {
return Err($crate::parser::error::ParseError::ExpectedToken(
vec![$($message.into(),)+],
Some(token.kind.to_string()),
token.span,
))
}
}
}};
([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )? => $out:expr),+ $(,)? ], $state:expr, $message:literal) => {
$crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); $out },)+ ], $state, [$message])
};
([ $($(|)? $( $pattern:pat_param )|+ $( if $guard: expr )?),+ $(,)? ], $state:expr, $message:literal) => {
$crate::peek_token!([ $($( $pattern )|+ $( if $guard )? => { $state.next(); },)+ ], $state, [$message])
$crate::expect_token!([ $($( $pattern )|+ $( if $guard )? => $out,)+ ], $state, [$message])
};
}

57
src/parser/mod.rs
View File

@ -143,7 +143,7 @@ impl Parser {
loop {
let name = self.ident(state)?;
expect_token!([TokenKind::Equals], state, "`=`");
self.skip(state, TokenKind::Equals)?;
let value = self.expression(state, Precedence::Lowest)?;
@ -297,7 +297,7 @@ impl Parser {
loop {
let key = self.ident(state)?;
expect_token!([TokenKind::Equals], state, "`=`");
self.skip(state, TokenKind::Equals)?;
let value = expect_literal!(state);
@ -320,7 +320,7 @@ impl Parser {
} else if state.current.kind == TokenKind::Colon {
self.colon(state)?;
let b = self.block(state, &TokenKind::EndDeclare)?;
expect_token!([TokenKind::EndDeclare], state, "`enddeclare`");
self.skip(state, TokenKind::EndDeclare)?;
self.semicolon(state)?;
b
} else {
@ -446,7 +446,7 @@ impl Parser {
let body = self.block(state, &TokenKind::RightBrace)?;
self.right_brace(state)?;
expect_token!([TokenKind::While], state, "`while`");
self.skip(state, TokenKind::While)?;
self.left_parenthesis(state)?;
let condition = self.expression(state, Precedence::Lowest)?;
@ -476,7 +476,7 @@ impl Parser {
if end_token == TokenKind::RightBrace {
self.right_brace(state)?;
} else {
expect_token!([TokenKind::EndWhile], state, "`endwhile`");
self.skip(state, TokenKind::EndWhile)?;
self.semicolon(state)?;
}
@ -548,7 +548,7 @@ impl Parser {
let then = self.block(state, &end_token)?;
if end_token == TokenKind::EndFor {
expect_token!([TokenKind::EndFor], state, "`endfor`");
self.skip(state, TokenKind::EndFor)?;
self.semicolon(state)?;
} else {
self.right_brace(state)?;
@ -568,7 +568,7 @@ impl Parser {
let expr = self.expression(state, Precedence::Lowest)?;
expect_token!([TokenKind::As], state, ["`as`"]);
self.skip(state, TokenKind::As)?;
let mut by_ref = state.current.kind == TokenKind::Ampersand;
if by_ref {
@ -604,7 +604,7 @@ impl Parser {
let body = self.block(state, &end_token)?;
if end_token == TokenKind::EndForeach {
expect_token!([TokenKind::EndForeach], state, "`endforeach`");
self.skip(state, TokenKind::EndForeach)?;
self.semicolon(state)?;
} else {
self.right_brace(state)?;
@ -643,11 +643,8 @@ impl Parser {
let condition = self.expression(state, Precedence::Lowest)?;
expect_token!(
[TokenKind::Colon, TokenKind::SemiColon],
state,
["`:`", "`;`"]
);
self.skip_any_of(state, &[TokenKind::Colon, TokenKind::SemiColon])?;
let mut body = Block::new();
while state.current.kind != TokenKind::Case
@ -666,11 +663,7 @@ impl Parser {
TokenKind::Default => {
state.next();
expect_token!(
[TokenKind::Colon, TokenKind::SemiColon],
state,
["`:`", "`;`"]
);
self.skip_any_of(state, &[TokenKind::Colon, TokenKind::SemiColon])?;
let mut body = Block::new();
@ -693,7 +686,7 @@ impl Parser {
}
if end_token == TokenKind::EndSwitch {
expect_token!([TokenKind::EndSwitch], state, ["`endswitch`"]);
self.skip(state, TokenKind::EndSwitch)?;
self.semicolon(state)?;
} else {
self.right_brace(state)?;
@ -760,7 +753,8 @@ impl Parser {
r#else = Some(body);
}
expect_token!([TokenKind::EndIf], state, ["`endif`"]);
self.skip(state, TokenKind::EndIf)?;
self.semicolon(state)?;
Statement::If {
@ -817,7 +811,7 @@ impl Parser {
});
}
expect_token!([TokenKind::Else], state, ["`else`"]);
self.skip(state, TokenKind::Else)?;
self.left_brace(state)?;
@ -1182,7 +1176,7 @@ impl Parser {
state.next();
}
expect_token!([TokenKind::DoubleArrow], state, "`=>`");
self.double_arrow(state)?;
let body = self.expression(state, Precedence::Lowest)?;
@ -1200,7 +1194,7 @@ impl Parser {
}
if !conditions.is_empty() {
expect_token!([TokenKind::DoubleArrow], state, "`=>`");
self.double_arrow(state)?;
} else {
break;
}
@ -1559,7 +1553,7 @@ impl Parser {
} else {
let index = self.expression(state, Precedence::Lowest)?;
expect_token!([TokenKind::RightBracket], state, ["`]`"]);
self.right_bracket(state)?;
Expression::ArrayIndex {
array: Box::new(lhs),
@ -1777,9 +1771,13 @@ impl Parser {
// FIXME: This feels hacky. We should probably produce different tokens from the lexer
// but since I already had the logic in place for parsing heredocs, this was
// the fastest way to get nowdocs working too.
let mut s =
expect_token!([TokenKind::StringPart(s) => s], state, "constant string");
let (indentation_type, indentation_amount) = expect_token!([TokenKind::EndDocString(_, indentation_type, indentation_amount) => (indentation_type, indentation_amount)], state, "label");
let mut s = expect_token!([
TokenKind::StringPart(s) => s
], state, "constant string");
let (indentation_type, indentation_amount) = expect_token!([
TokenKind::EndDocString(_, indentation_type, indentation_amount) => (indentation_type, indentation_amount)
], state, "label");
// FIXME: Hacky code, but it's late and I want to get this done.
if let Some(indentation_type) = indentation_type {
@ -1866,7 +1864,7 @@ impl Parser {
// "{$expr}"
state.next();
let e = self.expression(state, Precedence::Lowest)?;
expect_token!([TokenKind::RightBrace], state, "`}`");
self.right_brace(state)?;
Some(StringPart::Expr(Box::new(e)))
}
TokenKind::Variable(_) => {
@ -1912,7 +1910,8 @@ impl Parser {
}
};
expect_token!([TokenKind::RightBracket], state, "`]`");
self.right_bracket(state)?;
Expression::ArrayIndex {
array: Box::new(var),
index: Some(Box::new(index)),

14
src/parser/state.rs
View File

@ -180,6 +180,20 @@ impl State {
self.current.kind == TokenKind::Eof
}
pub fn pull(&mut self) -> Token {
let mut current: Token = Default::default();
std::mem::swap(&mut current, &mut self.current);
std::mem::swap(&mut self.current, &mut self.peek);
// peek already contains default.
if let Some(t) = self.iter.next() {
self.peek = t;
}
current
}
pub fn next(&mut self) {
// move peek to current
std::mem::swap(&mut self.current, &mut self.peek);