danog/PHP-Parser
1
0
Fork 0
mirror of https://github.com/danog/PHP-Parser.git synced 2024-11-30 04:19:30 +01:00
Code Issues Projects Releases Wiki Activity

Generalize the attribute generation for nodes

Browse Source 
Now two arrays are fetched from the lexer: $startAttributes and
$endAttributes. When constructing the attributes for a node, the
$startAttributes from the first token of the node and the $endAttributes
of the last token of the node are merged.

Now the end line is saved in the endLine attribute.
This commit is contained in:
nikic 2012-05-05 17:34:27 +02:00
parent 5438cc0d69
commit dd711f2a04
9 changed files with 503 additions and 443 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
grammar/kmyacc.php.parser
View File

@ -2,8 +2,8 @@
$meta # $meta #
#semval($) $this->yyval #semval($) $this->yyval
#semval($,%t) $this->yyval #semval($,%t) $this->yyval
#semval(%n) $this->yyastk[$this->yysp-(%l-%n)] #semval(%n) $this->yyastk[$this->stackPos-(%l-%n)]
#semval(%n,%t) $this->yyastk[$this->yysp-(%l-%n)] #semval(%n,%t) $this->yyastk[$this->stackPos-(%l-%n)]
#include; #include;
/* This is an automatically GENERATED file, which should not be manually edited. /* This is an automatically GENERATED file, which should not be manually edited.
@ -94,7 +94,7 @@ class #(-p)
protected $yyval; protected $yyval;
protected $yyastk; protected $yyastk;
protected $yysp; protected $stackPos;
protected $lexer; protected $lexer;
/** /**
@ -162,14 +162,29 @@ class #(-p)
public function parse($code) { public function parse($code) {
$this->lexer->startLexing($code); $this->lexer->startLexing($code);
$this->yysp = 0; // Stack pos // We start off with no lookahead-token
$stateStack = array($state = 0); // State stack
$this->yyastk = array(); // AST stack (?)
$yylstk = array($yyline = 1); // Line stack
$yydstk = array($yyDC = null); // Doc comment stack
$tokenId = self::TOKEN_NONE; $tokenId = self::TOKEN_NONE;
// The attributes for a node are taken from the first and last token of the node.
// From the first token only the startAttributes are taken and from the last only
// the endAttributes. Both are merged using the array union operator (+).
$startAttributes = array('startLine' => 1);
$endAttributes = array();
// In order to figure out the attributes for the starting token, we have to keep
// them in a stack
$attributeStack = array($startAttributes);
// Start off in the initial state and keep a stack of previous states
$state = 0;
$stateStack = array($state);
// AST stack (?)
$this->yyastk = array();
// Current position in the stack(s)
$this->stackPos = 0;
for (;;) { for (;;) {
#if -t #if -t
$this->YYTRACE_NEWSTATE($state, $tokenId); $this->YYTRACE_NEWSTATE($state, $tokenId);
@ -180,7 +195,7 @@ class #(-p)
} else { } else {
if ($tokenId === self::TOKEN_NONE) { if ($tokenId === self::TOKEN_NONE) {
// fetch the next token id from the lexer and fetch additional info by-ref // fetch the next token id from the lexer and fetch additional info by-ref
$origTokenId = $this->lexer->getNextToken($tokenValue, $yyline, $yyDC); $origTokenId = $this->lexer->getNextToken($tokenValue, $startAttributes, $endAttributes);
// map the lexer token id to the internally used token id's // map the lexer token id to the internally used token id's
$tokenId = $origTokenId >= 0 && $origTokenId < self::TOKEN_MAP_SIZE $tokenId = $origTokenId >= 0 && $origTokenId < self::TOKEN_MAP_SIZE
@ -194,8 +209,7 @@ class #(-p)
)); ));
} }
$yylstk[$this->yysp] = $yyline; $attributeStack[$this->stackPos] = $startAttributes;
$yydstk[$this->yysp] = $yyDC;
#if -t #if -t
$this->YYTRACE_READ($tokenId); $this->YYTRACE_READ($tokenId);
@ -222,12 +236,11 @@ class #(-p)
$this->YYTRACE_SHIFT($tokenId); $this->YYTRACE_SHIFT($tokenId);
#endif #endif
++$this->yysp; ++$this->stackPos;
$stateStack[$this->yysp] = $state = $yyn; $stateStack[$this->stackPos] = $state = $yyn;
$this->yyastk[$this->yysp] = $tokenValue; $this->yyastk[$this->stackPos] = $tokenValue;
$yylstk[$this->yysp] = $yyline; $attributeStack[$this->stackPos] = $startAttributes;
$yydstk[$this->yysp] = $yyDC;
$tokenId = self::TOKEN_NONE; $tokenId = self::TOKEN_NONE;
if ($yyn < self::YYNLSTATES) if ($yyn < self::YYNLSTATES)
@ -258,23 +271,21 @@ class #(-p)
#endif #endif
try { try {
$this->{'yyn' . $yyn}( $this->{'yyn' . $yyn}(
array( $attributeStack[$this->stackPos - self::$yylen[$yyn]]
'line' => $yylstk[$this->yysp - self::$yylen[$yyn]], + $endAttributes
'docComment' => $yydstk[$this->yysp - self::$yylen[$yyn]]
)
); );
} catch (PHPParser_Error $e) { } catch (PHPParser_Error $e) {
if (-1 === $e->getRawLine()) { if (-1 === $e->getRawLine()) {
$e->setRawLine($yyline); $e->setRawLine($startAttributes['startLine']);
} }
throw $e; throw $e;
} }
/* Goto - shift nonterminal */ /* Goto - shift nonterminal */
$this->yysp -= self::$yylen[$yyn]; $this->stackPos -= self::$yylen[$yyn];
$yyn = self::$yylhs[$yyn]; $yyn = self::$yylhs[$yyn];
if (($yyp = self::$yygbase[$yyn] + $stateStack[$this->yysp]) >= 0 if (($yyp = self::$yygbase[$yyn] + $stateStack[$this->stackPos]) >= 0
&& $yyp < self::YYGLAST && $yyp < self::YYGLAST
&& self::$yygcheck[$yyp] == $yyn) { && self::$yygcheck[$yyp] == $yyn) {
$state = self::$yygoto[$yyp]; $state = self::$yygoto[$yyp];
@ -282,17 +293,16 @@ class #(-p)
$state = self::$yygdefault[$yyn]; $state = self::$yygdefault[$yyn];
} }
++$this->yysp; ++$this->stackPos;
$stateStack[$this->yysp] = $state; $stateStack[$this->stackPos] = $state;
$this->yyastk[$this->yysp] = $this->yyval; $this->yyastk[$this->stackPos] = $this->yyval;
$yylstk[$this->yysp] = $yyline; $attributeStack[$this->stackPos] = $startAttributes;
$yydstk[$this->yysp] = $yyDC;
} else { } else {
/* error */ /* error */
throw new PHPParser_Error( throw new PHPParser_Error(
'Unexpected token ' . self::$terminals[$tokenId], 'Unexpected token ' . self::$terminals[$tokenId],
$yyline $startAttributes['startLine']
); );
} }
@ -312,7 +322,7 @@ class #(-p)
#noact #noact
protected function yyn%n() { protected function yyn%n() {
$this->yyval = $this->yyastk[$this->yysp]; $this->yyval = $this->yyastk[$this->stackPos];
} }
#endreduce #endreduce
#endif #endif

24
lib/PHPParser/Lexer.php
View File

@ -71,22 +71,24 @@ class PHPParser_Lexer
} }
/** /**
* Returns the next token id. * Fetches the next token.
* *
* @param mixed $value Variable to store token content in * @param mixed $value Variable to store token content in
* @param mixed $line Variable to store line in * @param mixed $startAttributes Variable to store start attributes in
* @param mixed $docComment Variable to store doc comment in * @param mixed $endAttributes Variable to store end attributes in
* *
* @return int Token id * @return int Token id
*/ */
public function getNextToken(&$value = null, &$line = null, &$docComment = null) { public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) {
$docComment = null; $startAttributes = array();
$endAttributes = array();
while (isset($this->tokens[++$this->pos])) { while (isset($this->tokens[++$this->pos])) {
$token = $this->tokens[$this->pos]; $token = $this->tokens[$this->pos];
if (is_string($token)) { if (is_string($token)) {
$line = $this->line; $startAttributes['startLine'] = $this->line;
$endAttributes['endLine'] = $this->line;
// bug in token_get_all // bug in token_get_all
if ('b"' === $token) { if ('b"' === $token) {
@ -100,15 +102,19 @@ class PHPParser_Lexer
$this->line += substr_count($token[1], "\n"); $this->line += substr_count($token[1], "\n");
if (T_DOC_COMMENT === $token[0]) { if (T_DOC_COMMENT === $token[0]) {
$docComment = $token[1]; $startAttributes['docComment'] = $token[1];
} elseif (!isset($this->dropTokens[$token[0]])) { } elseif (!isset($this->dropTokens[$token[0]])) {
$value = $token[1]; $value = $token[1];
$line = $token[2]; $startAttributes['startLine'] = $token[2];
$endAttributes['endLine'] = $this->line;
return $this->tokenMap[$token[0]]; return $this->tokenMap[$token[0]];
} }
} }
} }
$startAttributes['startLine'] = $this->line;
// 0 is the EOF token // 0 is the EOF token
return 0; return 0;
} }

4
lib/PHPParser/Lexer/Emulative.php
View File

@ -173,8 +173,8 @@ class PHPParser_Lexer_Emulative extends PHPParser_Lexer
} }
} }
public function getNextToken(&$value = null, &$line = null, &$docComment = null) { public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) {
$token = parent::getNextToken($value, $line, $docComment); $token = parent::getNextToken($value, $startAttributes, $endAttributes);
// replace new keywords by their respective tokens. This is not done // replace new keywords by their respective tokens. This is not done
// if we currently are in an object access (e.g. in $obj->namespace // if we currently are in an object access (e.g. in $obj->namespace

4
lib/PHPParser/NodeAbstract.php
View File

@ -40,7 +40,7 @@ abstract class PHPParser_NodeAbstract implements PHPParser_Node, IteratorAggrega
* @return int Line * @return int Line
*/ */
public function getLine() { public function getLine() {
return $this->getAttribute('line', -1); return $this->getAttribute('startLine', -1);
} }
/** /**
@ -49,7 +49,7 @@ abstract class PHPParser_NodeAbstract implements PHPParser_Node, IteratorAggrega
* @param int $line Line * @param int $line Line
*/ */
public function setLine($line) { public function setLine($line) {
$this->setAttribute('line', (int) $line); $this->setAttribute('startLine', (int) $line);
} }
/** /**

716
lib/PHPParser/Parser.php
View File

File diff suppressed because it is too large Load Diff

51
test/PHPParser/Tests/LexerTest.php
View File

@ -37,13 +37,13 @@ class PHPParser_Tests_LexerTest extends PHPUnit_Framework_TestCase
*/ */
public function testLex($code, $tokens) { public function testLex($code, $tokens) {
$this->lexer->startLexing($code); $this->lexer->startLexing($code);
while ($id = $this->lexer->getNextToken($value, $line, $docComment)) { while ($id = $this->lexer->getNextToken($value, $startAttributes, $endAttributes)) {
$token = array_shift($tokens); $token = array_shift($tokens);
$this->assertEquals($token[0], $id); $this->assertEquals($token[0], $id);
$this->assertEquals($token[1], $value); $this->assertEquals($token[1], $value);
$this->assertEquals($token[2], $line); $this->assertEquals($token[2], $startAttributes);
$this->assertEquals($token[3], $docComment); $this->assertEquals($token[3], $endAttributes);
} }
} }
@ -53,25 +53,56 @@ class PHPParser_Tests_LexerTest extends PHPUnit_Framework_TestCase
array( array(
'<?php tokens // ?>plaintext', '<?php tokens // ?>plaintext',
array( array(
array(PHPParser_Parser::T_STRING, 'tokens', 1, null), array(
array(ord(';'), '?>', 1, null), PHPParser_Parser::T_STRING, 'tokens',
array(PHPParser_Parser::T_INLINE_HTML, 'plaintext', 1, null), array('startLine' => 1), array('endLine' => 1)
),
array(
ord(';'), '?>',
array('startLine' => 1), array('endLine' => 1)
),
array(
PHPParser_Parser::T_INLINE_HTML, 'plaintext',
array('startLine' => 1), array('endLine' => 1)
),
) )
), ),
// tests line numbers // tests line numbers
array( array(
'<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $', '<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
array( array(
array(ord('$'), '$', 2, null), array(
array(PHPParser_Parser::T_STRING, 'token', 2, null), ord('$'), '$',
array(ord('$'), '$', 3, '/** doc' . "\n" . 'comment */') array('startLine' => 2), array('endLine' => 2)
),
array(
PHPParser_Parser::T_STRING, 'token',
array('startLine' => 2), array('endLine' => 2)
),
array(
ord('$'), '$',
array('startLine' => 3, 'docComment' => '/** doc' . "\n" . 'comment */'), array('endLine' => 3)
),
) )
), ),
// tests doccomment extraction // tests doccomment extraction
array( array(
'<?php /** docComment 1 *//** docComment 2 */ token', '<?php /** docComment 1 *//** docComment 2 */ token',
array( array(
array(PHPParser_Parser::T_STRING, 'token', 1, '/** docComment 2 */'), array(
PHPParser_Parser::T_STRING, 'token',
array('startLine' => 1, 'docComment' => '/** docComment 2 */'), array('endLine' => 1)
),
)
),
// tests differing start and end line
array(
'<?php "foo' . "\n" . 'bar"',
array(
array(
PHPParser_Parser::T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
array('startLine' => 1), array('endLine' => 2)
),
) )
), ),
); );

2
test/PHPParser/Tests/NodeAbstractTest.php
View File

@ -4,7 +4,7 @@ class PHPParser_Tests_NodeAbstractTest extends PHPUnit_Framework_TestCase
{ {
public function testConstruct() { public function testConstruct() {
$attributes = array( $attributes = array(
'line' => 10, 'startLine' => 10,
'docComment' => '/** doc comment */', 'docComment' => '/** doc comment */',
); );

4
test/PHPParser/Tests/NodeVisitor/NameResolverTest.php
View File

@ -202,8 +202,8 @@ EOC;
public function testAlreadyInUseError() { public function testAlreadyInUseError() {
$stmts = array( $stmts = array(
new PHPParser_Node_Stmt_Use(array( new PHPParser_Node_Stmt_Use(array(
new PHPParser_Node_Stmt_UseUse(new PHPParser_Node_Name('A\B'), 'B', array('line' => 1)), new PHPParser_Node_Stmt_UseUse(new PHPParser_Node_Name('A\B'), 'B', array('startLine' => 1)),
new PHPParser_Node_Stmt_UseUse(new PHPParser_Node_Name('C'), 'B', array('line' => 2)), new PHPParser_Node_Stmt_UseUse(new PHPParser_Node_Name('C'), 'B', array('startLine' => 2)),
)) ))
); );

69
test/PHPParser/Tests/Serializer/XMLTest.php
View File

@ -18,35 +18,38 @@ CODE;
<AST xmlns:node="http://nikic.github.com/PHPParser/XML/node" xmlns:subNode="http://nikic.github.com/PHPParser/XML/subNode" xmlns:attribute="http://nikic.github.com/PHPParser/XML/attribute" xmlns:scalar="http://nikic.github.com/PHPParser/XML/scalar"> <AST xmlns:node="http://nikic.github.com/PHPParser/XML/node" xmlns:subNode="http://nikic.github.com/PHPParser/XML/subNode" xmlns:attribute="http://nikic.github.com/PHPParser/XML/attribute" xmlns:scalar="http://nikic.github.com/PHPParser/XML/scalar">
<scalar:array> <scalar:array>
<node:Stmt_Function> <node:Stmt_Function>
<attribute:line>
<scalar:int>3</scalar:int>
</attribute:line>
<attribute:docComment> <attribute:docComment>
<scalar:string>/** doc comment */</scalar:string> <scalar:string>/** doc comment */</scalar:string>
</attribute:docComment> </attribute:docComment>
<attribute:startLine>
<scalar:int>3</scalar:int>
</attribute:startLine>
<attribute:endLine>
<scalar:int>5</scalar:int>
</attribute:endLine>
<subNode:byRef> <subNode:byRef>
<scalar:false/> <scalar:false/>
</subNode:byRef> </subNode:byRef>
<subNode:params> <subNode:params>
<scalar:array> <scalar:array>
<node:Param> <node:Param>
<attribute:line> <attribute:startLine>
<scalar:int>3</scalar:int> <scalar:int>3</scalar:int>
</attribute:line> </attribute:startLine>
<attribute:docComment> <attribute:endLine>
<scalar:null/> <scalar:int>3</scalar:int>
</attribute:docComment> </attribute:endLine>
<subNode:name> <subNode:name>
<scalar:string>a</scalar:string> <scalar:string>a</scalar:string>
</subNode:name> </subNode:name>
<subNode:default> <subNode:default>
<node:Scalar_LNumber> <node:Scalar_LNumber>
<attribute:line> <attribute:startLine>
<scalar:int>3</scalar:int> <scalar:int>3</scalar:int>
</attribute:line> </attribute:startLine>
<attribute:docComment> <attribute:endLine>
<scalar:null/> <scalar:int>3</scalar:int>
</attribute:docComment> </attribute:endLine>
<subNode:value> <subNode:value>
<scalar:int>0</scalar:int> <scalar:int>0</scalar:int>
</subNode:value> </subNode:value>
@ -60,23 +63,23 @@ CODE;
</subNode:byRef> </subNode:byRef>
</node:Param> </node:Param>
<node:Param> <node:Param>
<attribute:line> <attribute:startLine>
<scalar:int>3</scalar:int> <scalar:int>3</scalar:int>
</attribute:line> </attribute:startLine>
<attribute:docComment> <attribute:endLine>
<scalar:null/> <scalar:int>3</scalar:int>
</attribute:docComment> </attribute:endLine>
<subNode:name> <subNode:name>
<scalar:string>b</scalar:string> <scalar:string>b</scalar:string>
</subNode:name> </subNode:name>
<subNode:default> <subNode:default>
<node:Scalar_DNumber> <node:Scalar_DNumber>
<attribute:line> <attribute:startLine>
<scalar:int>3</scalar:int> <scalar:int>3</scalar:int>
</attribute:line> </attribute:startLine>
<attribute:docComment> <attribute:endLine>
<scalar:null/> <scalar:int>3</scalar:int>
</attribute:docComment> </attribute:endLine>
<subNode:value> <subNode:value>
<scalar:float>1</scalar:float> <scalar:float>1</scalar:float>
</subNode:value> </subNode:value>
@ -94,21 +97,21 @@ CODE;
<subNode:stmts> <subNode:stmts>
<scalar:array> <scalar:array>
<node:Stmt_Echo> <node:Stmt_Echo>
<attribute:line> <attribute:startLine>
<scalar:int>4</scalar:int> <scalar:int>4</scalar:int>
</attribute:line> </attribute:startLine>
<attribute:docComment> <attribute:endLine>
<scalar:null/> <scalar:int>4</scalar:int>
</attribute:docComment> </attribute:endLine>
<subNode:exprs> <subNode:exprs>
<scalar:array> <scalar:array>
<node:Scalar_String> <node:Scalar_String>
<attribute:line> <attribute:startLine>
<scalar:int>4</scalar:int> <scalar:int>4</scalar:int>
</attribute:line> </attribute:startLine>
<attribute:docComment> <attribute:endLine>
<scalar:null/> <scalar:int>4</scalar:int>
</attribute:docComment> </attribute:endLine>
<subNode:value> <subNode:value>
<scalar:string>Foo</scalar:string> <scalar:string>Foo</scalar:string>
</subNode:value> </subNode:value>