From 17c5307e4f330d4fe16943538797c7f60a371ef4 Mon Sep 17 00:00:00 2001 From: Ryan Chandler Date: Thu, 15 Sep 2022 00:09:01 +0100 Subject: [PATCH 1/7] parser: add test cases for foreach refs --- trunk_parser/src/parser/mod.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/trunk_parser/src/parser/mod.rs b/trunk_parser/src/parser/mod.rs index 52adac5..5285098 100644 --- a/trunk_parser/src/parser/mod.rs +++ b/trunk_parser/src/parser/mod.rs @@ -3721,6 +3721,32 @@ mod tests { ); } + #[test] + fn simple_foreach_reference() { + assert_ast(" &$c) {}", &[ + Statement::Foreach { + expr: Expression::Variable { name: "a".into() }, + by_ref: true, + key_var: Some(Expression::Variable { name: "b".into() }), + value_var: Expression::Variable { name: "c".into() }, + body: vec![] + } + ]); + } + fn assert_ast(source: &str, expected: &[Statement]) { let mut lexer = Lexer::new(None); let tokens = lexer.tokenize(source).unwrap(); From 2d09a251d330ee94ee76dc140faa14f978486f3f Mon Sep 17 00:00:00 2001 From: Ryan Chandler Date: Thu, 15 Sep 2022 00:15:31 +0100 Subject: [PATCH 2/7] parser: support function params being marked as refs --- trunk_parser/src/ast.rs | 8 ++++ trunk_parser/src/parser/mod.rs | 72 ++++++++++++++++++++++++++----- trunk_parser/src/parser/params.rs | 21 ++++++--- 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/trunk_parser/src/ast.rs b/trunk_parser/src/ast.rs index 6377dab..f452124 100644 --- a/trunk_parser/src/ast.rs +++ b/trunk_parser/src/ast.rs @@ -35,6 +35,12 @@ impl From<&[u8]> for Identifier { } } +impl From<&str> for Identifier { + fn from(name: &str) -> Self { + Self::from(ByteString::from(name)) + } +} + pub type ParamList = Vec; #[derive(Debug, PartialEq, Clone)] @@ -44,6 +50,7 @@ pub struct Param { pub variadic: bool, pub default: Option, pub flag: Option, + pub by_ref: bool, } impl From for Param { @@ -54,6 +61,7 @@ impl From for Param { variadic: false, default: None, flag: None, + by_ref: false, } } } diff --git a/trunk_parser/src/parser/mod.rs b/trunk_parser/src/parser/mod.rs index 5285098..e4e6197 100644 --- a/trunk_parser/src/parser/mod.rs +++ b/trunk_parser/src/parser/mod.rs @@ -2906,6 +2906,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -2925,6 +2926,7 @@ mod tests { variadic: true, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -2941,6 +2943,7 @@ mod tests { variadic: true, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -2958,6 +2961,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }, Param { name: Expression::Variable { name: "baz".into() }, @@ -2965,6 +2969,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }, Param { name: Expression::Variable { name: "car".into() }, @@ -2972,6 +2977,7 @@ mod tests { variadic: true, default: None, flag: None, + by_ref: false, }, ], body: vec![], @@ -2992,6 +2998,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -3011,6 +3018,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -3031,6 +3039,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -3050,6 +3059,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -3070,6 +3080,7 @@ mod tests { variadic: false, default: None, flag: None, + by_ref: false, }], body: vec![], return_type: None, @@ -3723,28 +3734,69 @@ mod tests { #[test] fn simple_foreach_reference() { - assert_ast(" &$c) {}", &[ - Statement::Foreach { + assert_ast( + " &$c) {}", + &[Statement::Foreach { expr: Expression::Variable { name: "a".into() }, by_ref: true, key_var: Some(Expression::Variable { name: "b".into() }), value_var: Expression::Variable { name: "c".into() }, - body: vec![] - } - ]); + body: vec![], + }], + ); + } + + #[test] + fn function_with_ref_param() { + assert_ast( + " null;", + &[expr!(Expression::ArrowFunction { + params: vec![Param { + name: Expression::Variable { name: "b".into() }, + r#type: None, + variadic: false, + flag: None, + default: None, + by_ref: true, + }], + return_type: None, + expr: Box::new(Expression::Null) + })], + ); } fn assert_ast(source: &str, expected: &[Statement]) { diff --git a/trunk_parser/src/parser/params.rs b/trunk_parser/src/parser/params.rs index b3e1a7f..6a5af9f 100644 --- a/trunk_parser/src/parser/params.rs +++ b/trunk_parser/src/parser/params.rs @@ -27,18 +27,26 @@ impl Parser { // 1. If we don't see a variable, we should expect a type-string. if !matches!( self.current.kind, - TokenKind::Variable(_) | TokenKind::Ellipsis + TokenKind::Variable(_) | TokenKind::Ellipsis | TokenKind::Ampersand ) || self.config.force_type_strings { // 1a. Try to parse the type. param_type = Some(self.type_string()?); } - let variadic = if self.current.kind == TokenKind::Ellipsis { - self.next(); - true - } else { - false + let mut variadic = false; + let mut by_ref = false; + + match self.current.kind { + TokenKind::Ellipsis => { + self.next(); + variadic = true; + } + TokenKind::Ampersand => { + self.next(); + by_ref = true; + } + _ => {} }; // 2. Then expect a variable. @@ -57,6 +65,7 @@ impl Parser { variadic, default, flag, + by_ref, }); self.optional_comma()?; From 630ad85f34ddb8fba35c0d1cb85292b357668549 Mon Sep 17 00:00:00 2001 From: Ryan Chandler Date: Thu, 15 Sep 2022 00:42:28 +0100 Subject: [PATCH 3/7] parser: support named functions returning refs --- trunk_parser/src/ast.rs | 2 + trunk_parser/src/parser/mod.rs | 82 ++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/trunk_parser/src/ast.rs b/trunk_parser/src/ast.rs index f452124..c1304d3 100644 --- a/trunk_parser/src/ast.rs +++ b/trunk_parser/src/ast.rs @@ -233,6 +233,7 @@ pub enum Statement { params: Vec, body: Block, return_type: Option, + by_ref: bool, }, Class { name: Identifier, @@ -259,6 +260,7 @@ pub enum Statement { body: Block, flags: Vec, return_type: Option, + by_ref: bool, }, If { condition: Expression, diff --git a/trunk_parser/src/parser/mod.rs b/trunk_parser/src/parser/mod.rs index e4e6197..fbf83ee 100644 --- a/trunk_parser/src/parser/mod.rs +++ b/trunk_parser/src/parser/mod.rs @@ -531,6 +531,7 @@ impl Parser { body: vec![], return_type, flags: vec![MethodFlag::Public], + by_ref: false, }) } TokenKind::Function => { @@ -562,6 +563,7 @@ impl Parser { body: vec![], return_type, flags: vec![], + by_ref: false, }) } _ => { @@ -934,8 +936,28 @@ impl Parser { ret } } - TokenKind::Function if matches!(self.peek.kind, TokenKind::Identifier(_)) => { - self.function()? + TokenKind::Function if matches!(self.peek.kind, TokenKind::Identifier(_) | TokenKind::Ampersand) => { + // FIXME: This is incredibly hacky but we don't have a way to look at + // the next N tokens right now. We could probably do with a `peek_buf()` + // method like the Lexer has. + if self.peek.kind == TokenKind::Ampersand { + let mut cloned = self.iter.clone(); + for (index, _) in self.iter.clone().enumerate() { + if ! matches!(cloned.nth(index), Some(Token { kind: TokenKind::Identifier(_), .. })) { + let expr = self.expression(Precedence::Lowest)?; + + self.semi()?; + + return Ok(Statement::Expression { expr }); + } + + break; + } + + self.function()? + } else { + self.function()? + } } TokenKind::SemiColon => { self.next(); @@ -1053,6 +1075,13 @@ impl Parser { fn function(&mut self) -> ParseResult { self.next(); + let by_ref = if self.current.kind == TokenKind::Ampersand { + self.next(); + true + } else { + false + }; + let name = self.ident()?; self.lparen()?; @@ -1080,6 +1109,7 @@ impl Parser { params, body, return_type, + by_ref, }) } @@ -1264,6 +1294,13 @@ impl Parser { if flags.contains(&TokenKind::Abstract) { self.next(); + let by_ref = if self.current.kind == TokenKind::Ampersand { + self.next(); + true + } else { + false + }; + let name = self.ident()?; self.lparen()?; @@ -1290,6 +1327,7 @@ impl Parser { body: vec![], return_type, flags: flags.iter().map(|t| t.clone().into()).collect(), + by_ref, }) } else { match self.function()? { @@ -1298,12 +1336,14 @@ impl Parser { params, body, return_type, + by_ref, } => Ok(Statement::Method { name, params, body, flags: flags.iter().map(|t| t.clone().into()).collect(), return_type, + by_ref, }), _ => unreachable!(), } @@ -1365,12 +1405,14 @@ impl Parser { params, body, return_type, + by_ref } => Ok(Statement::Method { name, params, body, flags: vec![], return_type, + by_ref }), _ => unreachable!(), }, @@ -2272,6 +2314,7 @@ mod tests { .collect::>(), body: $body.to_vec(), return_type: None, + by_ref: false, } }; } @@ -2318,6 +2361,7 @@ mod tests { flags: $flags.to_vec(), body: $body.to_vec(), return_type: None, + by_ref: false, } }; } @@ -2910,6 +2954,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); } @@ -2930,6 +2975,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); @@ -2947,6 +2993,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); @@ -2982,6 +3029,7 @@ mod tests { ], body: vec![], return_type: None, + by_ref: false, }], ); } @@ -3002,6 +3050,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); } @@ -3022,6 +3071,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); @@ -3043,6 +3093,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); } @@ -3063,6 +3114,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); @@ -3084,6 +3136,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); } @@ -3097,6 +3150,7 @@ mod tests { params: vec![], body: vec![], return_type: Some(Type::Plain("string".into())), + by_ref: false, }], ); @@ -3107,6 +3161,7 @@ mod tests { params: vec![], body: vec![], return_type: Some(Type::Void), + by_ref: false, }], ); } @@ -3188,7 +3243,8 @@ mod tests { params: vec![], body: vec![], return_type: None, - flags: vec![MethodFlag::Public,] + flags: vec![MethodFlag::Public,], + by_ref: false, }] }), args: vec![] @@ -3687,6 +3743,7 @@ mod tests { params: vec![], body: vec![], return_type: None, + by_ref: false, }], }], ); @@ -3776,6 +3833,7 @@ mod tests { }], body: vec![], return_type: None, + by_ref: false, }], ); } @@ -3799,6 +3857,24 @@ mod tests { ); } + #[test] + fn function_returning_ref() { + assert_ast(" Date: Thu, 15 Sep 2022 00:45:33 +0100 Subject: [PATCH 4/7] parser: support closures returning by ref --- trunk_parser/src/ast.rs | 1 + trunk_parser/src/parser/mod.rs | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/trunk_parser/src/ast.rs b/trunk_parser/src/ast.rs index c1304d3..41c5735 100644 --- a/trunk_parser/src/ast.rs +++ b/trunk_parser/src/ast.rs @@ -453,6 +453,7 @@ pub enum Expression { return_type: Option, body: Block, r#static: bool, + by_ref: bool, }, ArrowFunction { params: Vec, diff --git a/trunk_parser/src/parser/mod.rs b/trunk_parser/src/parser/mod.rs index fbf83ee..358b6f6 100644 --- a/trunk_parser/src/parser/mod.rs +++ b/trunk_parser/src/parser/mod.rs @@ -1630,12 +1630,14 @@ impl Parser { uses, return_type, body, + by_ref, .. } => Expression::Closure { params, uses, return_type, body, + by_ref, r#static: true, }, _ => unreachable!(), @@ -1644,6 +1646,13 @@ impl Parser { TokenKind::Function => { self.next(); + let by_ref = if self.current.kind == TokenKind::Ampersand { + self.next(); + true + } else { + false + }; + self.lparen()?; let params = self.param_list()?; @@ -1715,6 +1724,7 @@ impl Parser { return_type, body, r#static: false, + by_ref, } } TokenKind::Fn => { @@ -3758,7 +3768,8 @@ mod tests { uses: vec![], return_type: None, body: vec![], - r#static: false + r#static: false, + by_ref: false, })], ); } @@ -3784,7 +3795,8 @@ mod tests { uses: vec![], return_type: None, body: vec![], - r#static: true + r#static: true, + by_ref: false, })], ); } @@ -3875,6 +3887,20 @@ mod tests { }]); } + #[test] + fn closure_returning_ref() { + assert_ast(" Date: Thu, 15 Sep 2022 00:46:13 +0100 Subject: [PATCH 5/7] parser: add test for static closures returning by ref --- trunk_parser/src/parser/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/trunk_parser/src/parser/mod.rs b/trunk_parser/src/parser/mod.rs index 358b6f6..f20f9ad 100644 --- a/trunk_parser/src/parser/mod.rs +++ b/trunk_parser/src/parser/mod.rs @@ -3901,6 +3901,20 @@ mod tests { ]); } + #[test] + fn static_closures_returning_by_ref() { + assert_ast(" Date: Thu, 15 Sep 2022 00:47:51 +0100 Subject: [PATCH 6/7] parser: support arrow functions returning by ref --- trunk_parser/src/ast.rs | 1 + trunk_parser/src/parser/mod.rs | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/trunk_parser/src/ast.rs b/trunk_parser/src/ast.rs index 41c5735..b30d12b 100644 --- a/trunk_parser/src/ast.rs +++ b/trunk_parser/src/ast.rs @@ -459,6 +459,7 @@ pub enum Expression { params: Vec, return_type: Option, expr: Box, + by_ref: bool, }, New { target: Box, diff --git a/trunk_parser/src/parser/mod.rs b/trunk_parser/src/parser/mod.rs index f20f9ad..e9d4f4f 100644 --- a/trunk_parser/src/parser/mod.rs +++ b/trunk_parser/src/parser/mod.rs @@ -1730,6 +1730,13 @@ impl Parser { TokenKind::Fn => { self.next(); + let by_ref = if self.current.kind == TokenKind::Ampersand { + self.next(); + true + } else { + false + }; + self.lparen()?; let params = self.param_list()?; @@ -1752,6 +1759,7 @@ impl Parser { params, return_type, expr: Box::new(value), + by_ref, } } TokenKind::New => { @@ -3781,7 +3789,8 @@ mod tests { &[expr!(Expression::ArrowFunction { params: vec![], return_type: None, - expr: Box::new(Expression::Null) + expr: Box::new(Expression::Null), + by_ref: false, })], ); } @@ -3864,7 +3873,8 @@ mod tests { by_ref: true, }], return_type: None, - expr: Box::new(Expression::Null) + expr: Box::new(Expression::Null), + by_ref: false, })], ); } @@ -3915,6 +3925,18 @@ mod tests { ]); } + #[test] + fn arrow_functions_returning_by_ref() { + assert_ast(" null;", &[ + expr!(Expression::ArrowFunction { + params: vec![], + expr: Box::new(Expression::Null), + return_type: None, + by_ref: true, + }) + ]); + } + fn assert_ast(source: &str, expected: &[Statement]) { let mut lexer = Lexer::new(None); let tokens = lexer.tokenize(source).unwrap(); From 17a48e3ce2982c8d03679de7987a93f9b53de3f7 Mon Sep 17 00:00:00 2001 From: Ryan Chandler Date: Thu, 15 Sep 2022 00:48:59 +0100 Subject: [PATCH 7/7] chore: format --- trunk_parser/src/parser/mod.rs | 75 +++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/trunk_parser/src/parser/mod.rs b/trunk_parser/src/parser/mod.rs index e9d4f4f..b548d42 100644 --- a/trunk_parser/src/parser/mod.rs +++ b/trunk_parser/src/parser/mod.rs @@ -936,14 +936,25 @@ impl Parser { ret } } - TokenKind::Function if matches!(self.peek.kind, TokenKind::Identifier(_) | TokenKind::Ampersand) => { + TokenKind::Function + if matches!( + self.peek.kind, + TokenKind::Identifier(_) | TokenKind::Ampersand + ) => + { // FIXME: This is incredibly hacky but we don't have a way to look at // the next N tokens right now. We could probably do with a `peek_buf()` // method like the Lexer has. if self.peek.kind == TokenKind::Ampersand { let mut cloned = self.iter.clone(); for (index, _) in self.iter.clone().enumerate() { - if ! matches!(cloned.nth(index), Some(Token { kind: TokenKind::Identifier(_), .. })) { + if !matches!( + cloned.nth(index), + Some(Token { + kind: TokenKind::Identifier(_), + .. + }) + ) { let expr = self.expression(Precedence::Lowest)?; self.semi()?; @@ -1405,14 +1416,14 @@ impl Parser { params, body, return_type, - by_ref + by_ref, } => Ok(Statement::Method { name, params, body, flags: vec![], return_type, - by_ref + by_ref, }), _ => unreachable!(), }, @@ -3881,60 +3892,66 @@ mod tests { #[test] fn function_returning_ref() { - assert_ast(" null;", &[ - expr!(Expression::ArrowFunction { + assert_ast( + " null;", + &[expr!(Expression::ArrowFunction { params: vec![], expr: Box::new(Expression::Null), return_type: None, by_ref: true, - }) - ]); + })], + ); } fn assert_ast(source: &str, expected: &[Statement]) {