phabel/php-parser
1
0
Fork 0
mirror of https://github.com/phabelio/PHP-Parser.git synced 2024-11-26 20:14:46 +01:00
Code Issues Projects Releases Wiki Activity

Switch to a normalized token representation

Browse Source 
Each token is now represented by a Token object.
This commit is contained in:
Nikita Popov 2019-06-30 13:10:12 +02:00
parent 3f718ee2c3
commit 6f74784e16
10 changed files with 198 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
lib/PhpParser/Internal/TokenStream.php
View File

@ -2,6 +2,8 @@
namespace PhpParser\Internal;
use PhpParser\Token;
/**
* Provides operations on token streams, for use by pretty printer.
*
@ -9,7 +11,7 @@ namespace PhpParser\Internal;
*/
class TokenStream
{
/** @var array Tokens (in token_get_all format) */
/** @var Token[] */
private $tokens;
/** @var int[] Map from position to indentation */
private $indentMap;
@ -17,7 +19,7 @@ class TokenStream
/**
* Create token stream instance.
*
* @param array $tokens Tokens in token_get_all() format
* @param Token[] $tokens Tokens
*/
public function __construct(array $tokens) {
$this->tokens = $tokens;
@ -33,8 +35,8 @@ class TokenStream
* @return bool
*/
public function haveParens(int $startPos, int $endPos) : bool {
return $this->haveTokenImmediativelyBefore($startPos, '(')
&& $this->haveTokenImmediatelyAfter($endPos, ')');
return $this->haveTokenImmediativelyBefore($startPos, \ord('('))
&& $this->haveTokenImmediatelyAfter($endPos, \ord(')'));
}
/**
@ -46,8 +48,8 @@ class TokenStream
* @return bool
*/
public function haveBraces(int $startPos, int $endPos) : bool {
return $this->haveTokenImmediativelyBefore($startPos, '{')
&& $this->haveTokenImmediatelyAfter($endPos, '}');
return $this->haveTokenImmediativelyBefore($startPos, \ord('{'))
&& $this->haveTokenImmediatelyAfter($endPos, \ord('}'));
}
/**
@ -64,7 +66,7 @@ class TokenStream
$tokens = $this->tokens;
$pos--;
for (; $pos >= 0; $pos--) {
$tokenType = $tokens[$pos][0];
$tokenType = $tokens[$pos]->id;
if ($tokenType === $expectedTokenType) {
return true;
}
@ -90,7 +92,7 @@ class TokenStream
$tokens = $this->tokens;
$pos++;
for (; $pos < \count($tokens); $pos++) {
$tokenType = $tokens[$pos][0];
$tokenType = $tokens[$pos]->id;
if ($tokenType === $expectedTokenType) {
return true;
}
@ -110,7 +112,7 @@ class TokenStream
return $pos;
}
if ($tokens[$pos][0] !== $skipTokenType) {
if ($tokens[$pos]->id !== $skipTokenType) {
// Shouldn't happen. The skip token MUST be there
throw new \Exception('Encountered unexpected token');
}
@ -127,7 +129,7 @@ class TokenStream
return $pos;
}
if ($tokens[$pos][0] !== $skipTokenType) {
if ($tokens[$pos]->id !== $skipTokenType) {
// Shouldn't happen. The skip token MUST be there
throw new \Exception('Encountered unexpected token');
}
@ -145,7 +147,7 @@ class TokenStream
public function skipLeftWhitespace(int $pos) {
$tokens = $this->tokens;
for (; $pos >= 0; $pos--) {
$type = $tokens[$pos][0];
$type = $tokens[$pos]->id;
if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
break;
}
@ -162,7 +164,7 @@ class TokenStream
public function skipRightWhitespace(int $pos) {
$tokens = $this->tokens;
for ($count = \count($tokens); $pos < $count; $pos++) {
$type = $tokens[$pos][0];
$type = $tokens[$pos]->id;
if ($type !== \T_WHITESPACE && $type !== \T_COMMENT && $type !== \T_DOC_COMMENT) {
break;
}
@ -173,7 +175,7 @@ class TokenStream
public function findRight($pos, $findTokenType) {
$tokens = $this->tokens;
for ($count = \count($tokens); $pos < $count; $pos++) {
$type = $tokens[$pos][0];
$type = $tokens[$pos]->id;
if ($type === $findTokenType) {
return $pos;
}
@ -206,23 +208,19 @@ class TokenStream
$result = '';
for ($pos = $from; $pos < $to; $pos++) {
$token = $tokens[$pos];
if (\is_array($token)) {
$type = $token[0];
$content = $token[1];
if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
$result .= $content;
} else {
// TODO Handle non-space indentation
if ($indent < 0) {
$result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
} elseif ($indent > 0) {
$result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
} else {
$result .= $content;
}
}
$type = $token->id;
$content = $token->value;
if ($type === \T_CONSTANT_ENCAPSED_STRING || $type === \T_ENCAPSED_AND_WHITESPACE) {
$result .= $content;
} else {
$result .= $token;
// TODO Handle non-space indentation
if ($indent < 0) {
$result .= str_replace("\n" . str_repeat(" ", -$indent), "\n", $content);
} elseif ($indent > 0) {
$result .= str_replace("\n", "\n" . str_repeat(" ", $indent), $content);
} else {
$result .= $content;
}
}
}
return $result;
@ -239,8 +237,8 @@ class TokenStream
foreach ($this->tokens as $token) {
$indentMap[] = $indent;
if ($token[0] === \T_WHITESPACE) {
$content = $token[1];
if ($token->id === \T_WHITESPACE) {
$content = $token->value;
$newlinePos = \strrpos($content, "\n");
if (false !== $newlinePos) {
$indent = \strlen($content) - $newlinePos - 1;

191
lib/PhpParser/Lexer.php
View File

@ -11,11 +11,11 @@ class Lexer
* file positions from going out of sync. */
const T_BAD_CHARACTER = -1;
/** @var string */
protected $code;
/** @var Token[] */
protected $tokens;
protected $pos;
protected $line;
protected $filePos;
protected $prevCloseTagHasNewline;
protected $tokenMap;
@ -87,9 +87,7 @@ class Lexer
$scream = ini_set('xdebug.scream', '0');
error_clear_last();
$this->tokens = @token_get_all($code);
$this->handleErrors($errorHandler);
$this->tokens = $this->createNormalizedTokens($code, $errorHandler);
if (false !== $scream) {
ini_set('xdebug.scream', $scream);
@ -109,7 +107,7 @@ class Lexer
);
}
$tokens[] = [self::T_BAD_CHARACTER, $chr, $line];
$tokens[] = new Token(self::T_BAD_CHARACTER, $chr, $line, $i);
$errorHandler->handleError(new Error($errorMsg, [
'startLine' => $line,
'endLine' => $line,
@ -125,10 +123,10 @@ class Lexer
*
* @return bool
*/
private function isUnterminatedComment($token) : bool {
return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
&& substr($token[1], 0, 2) === '/*'
&& substr($token[1], -2) !== '*/';
private function isUnterminatedComment(Token $token) : bool {
return ($token->id === \T_COMMENT || $token->id === \T_DOC_COMMENT)
&& substr($token->value, 0, 2) === '/*'
&& substr($token->value, -2) !== '*/';
}
/**
@ -146,75 +144,65 @@ class Lexer
return null !== error_get_last();
}
protected function handleErrors(ErrorHandler $errorHandler) {
if (!$this->errorMayHaveOccurred()) {
return;
}
// PHP's error handling for token_get_all() is rather bad, so if we want detailed
// error information we need to compute it ourselves. Invalid character errors are
// detected by finding "gaps" in the token array. Unterminated comments are detected
// by checking if a trailing comment has a "*/" at the end.
private function createNormalizedTokens(string $code, ErrorHandler $errorHandler) {
error_clear_last();
$rawTokens = @token_get_all($code);
$checkForMissingTokens = $this->errorMayHaveOccurred();
$tokens = [];
$filePos = 0;
$line = 1;
$numTokens = \count($this->tokens);
for ($i = 0; $i < $numTokens; $i++) {
$token = $this->tokens[$i];
$tokenValue = \is_string($token) ? $token : $token[1];
$tokenLen = \strlen($tokenValue);
foreach ($rawTokens as $rawToken) {
if (\is_array($rawToken)) {
$token = new Token($rawToken[0], $rawToken[1], $line, $filePos);
} elseif (\strlen($rawToken) == 2) {
// Bug in token_get_all() when lexing b".
$token = new Token(\ord('"'), $rawToken, $line, $filePos);
} else {
$token = new Token(\ord($rawToken), $rawToken, $line, $filePos);
}
if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
$value = $token->value;
$tokenLen = \strlen($value);
if ($checkForMissingTokens && substr($code, $filePos, $tokenLen) !== $value) {
// Something is missing, must be an invalid character
$nextFilePos = strpos($this->code, $tokenValue, $filePos);
$nextFilePos = strpos($code, $value, $filePos);
$badCharTokens = $this->handleInvalidCharacterRange(
$filePos, $nextFilePos, $line, $errorHandler);
$tokens = array_merge($tokens, $badCharTokens);
$filePos = (int) $nextFilePos;
array_splice($this->tokens, $i, 0, $badCharTokens);
$numTokens += \count($badCharTokens);
$i += \count($badCharTokens);
}
$tokens[] = $token;
$filePos += $tokenLen;
$line += substr_count($tokenValue, "\n");
$line += substr_count($value, "\n");
}
if ($filePos !== \strlen($this->code)) {
if (substr($this->code, $filePos, 2) === '/*') {
// Unlike PHP, HHVM will drop unterminated comments entirely
$comment = substr($this->code, $filePos);
$errorHandler->handleError(new Error('Unterminated comment', [
'startLine' => $line,
'endLine' => $line + substr_count($comment, "\n"),
'startFilePos' => $filePos,
'endFilePos' => $filePos + \strlen($comment),
]));
// Emulate the PHP behavior
$isDocComment = isset($comment[3]) && $comment[3] === '*';
$this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
} else {
// Invalid characters at the end of the input
$badCharTokens = $this->handleInvalidCharacterRange(
$filePos, \strlen($this->code), $line, $errorHandler);
$this->tokens = array_merge($this->tokens, $badCharTokens);
}
return;
if ($filePos !== \strlen($code)) {
// Invalid characters at the end of the input
$badCharTokens = $this->handleInvalidCharacterRange(
$filePos, \strlen($code), $line, $errorHandler);
$tokens = array_merge($tokens, $badCharTokens);
}
if (count($this->tokens) > 0) {
if (\count($tokens) > 0) {
// Check for unterminated comment
$lastToken = $this->tokens[count($this->tokens) - 1];
$lastToken = $tokens[\count($tokens) - 1];
if ($this->isUnterminatedComment($lastToken)) {
$errorHandler->handleError(new Error('Unterminated comment', [
'startLine' => $line - substr_count($lastToken[1], "\n"),
'startLine' => $line - substr_count($lastToken->value, "\n"),
'endLine' => $line,
'startFilePos' => $filePos - \strlen($lastToken[1]),
'startFilePos' => $filePos - \strlen($lastToken->value),
'endFilePos' => $filePos,
]));
}
}
// Add an EOF sentinel token
// TODO: Should the value be an empty string instead?
$tokens[] = new Token(0, "\0", $line, \strlen($code));
return $tokens;
}
/**
@ -244,70 +232,49 @@ class Lexer
$endAttributes = [];
while (1) {
if (isset($this->tokens[++$this->pos])) {
$token = $this->tokens[$this->pos];
} else {
// EOF token with ID 0
$token = "\0";
}
$token = $this->tokens[++$this->pos];
if ($this->attributeStartLineUsed) {
$startAttributes['startLine'] = $this->line;
$startAttributes['startLine'] = $token->line;
}
if ($this->attributeStartTokenPosUsed) {
$startAttributes['startTokenPos'] = $this->pos;
}
if ($this->attributeStartFilePosUsed) {
$startAttributes['startFilePos'] = $this->filePos;
$startAttributes['startFilePos'] = $token->filePos;
}
if (\is_string($token)) {
$value = $token;
if (isset($token[1])) {
// bug in token_get_all
$this->filePos += 2;
$id = ord('"');
} else {
$this->filePos += 1;
$id = ord($token);
}
} elseif (!isset($this->dropTokens[$token[0]])) {
$value = $token[1];
$id = $this->tokenMap[$token[0]];
if (\T_CLOSE_TAG === $token[0]) {
$this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
} elseif (\T_INLINE_HTML === $token[0]) {
$phpId = $token->id;
$value = $token->value;
if (!isset($this->dropTokens[$phpId])) {
$id = $this->tokenMap[$phpId];
if (\T_CLOSE_TAG === $phpId) {
$this->prevCloseTagHasNewline = false !== strpos($value, "\n");
} elseif (\T_INLINE_HTML === $phpId) {
$startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
}
$this->line += substr_count($value, "\n");
$this->filePos += \strlen($value);
} else {
if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
if ($this->attributeCommentsUsed) {
$comment = \T_DOC_COMMENT === $token[0]
? new Comment\Doc($token[1], $this->line, $this->filePos, $this->pos)
: new Comment($token[1], $this->line, $this->filePos, $this->pos);
$startAttributes['comments'][] = $comment;
}
if ($this->attributeEndLineUsed) {
$endAttributes['endLine'] = $token->line + substr_count($value, "\n");
}
if ($this->attributeEndTokenPosUsed) {
$endAttributes['endTokenPos'] = $this->pos;
}
if ($this->attributeEndFilePosUsed) {
$endAttributes['endFilePos'] = $token->filePos + \strlen($value) - 1;
}
$this->line += substr_count($token[1], "\n");
$this->filePos += \strlen($token[1]);
continue;
return $id;
}
if ($this->attributeEndLineUsed) {
$endAttributes['endLine'] = $this->line;
if (\T_COMMENT === $phpId || \T_DOC_COMMENT === $phpId) {
if ($this->attributeCommentsUsed) {
$comment = \T_DOC_COMMENT === $phpId
? new Comment\Doc($value, $token->line, $token->filePos, $this->pos)
: new Comment($value, $token->line, $token->filePos, $this->pos);
$startAttributes['comments'][] = $comment;
}
}
if ($this->attributeEndTokenPosUsed) {
$endAttributes['endTokenPos'] = $this->pos;
}
if ($this->attributeEndFilePosUsed) {
$endAttributes['endFilePos'] = $this->filePos - 1;
}
return $id;
}
throw new \RuntimeException('Reached end of lexer loop');
@ -333,8 +300,9 @@ class Lexer
* @return string Remaining text
*/
public function handleHaltCompiler() : string {
// text after T_HALT_COMPILER, still including ();
$textAfter = substr($this->code, $this->filePos);
// Text after T_HALT_COMPILER, still including ();
$tokenAfter = $this->tokens[$this->pos + 1];
$textAfter = substr($this->code, $tokenAfter->filePos);
// ensure that it is followed by ();
// this simplifies the situation, by not allowing any comments
@ -343,8 +311,8 @@ class Lexer
throw new Error('__HALT_COMPILER must be followed by "();"');
}
// prevent the lexer from returning any further tokens
$this->pos = count($this->tokens);
// Point to one before EOF token, so it will be returned on the getNextToken() call
$this->pos = count($this->tokens) - 2;
// return with (); removed
return substr($textAfter, strlen($matches[0]));
@ -362,9 +330,12 @@ class Lexer
protected function createTokenMap() : array {
$tokenMap = [];
// 256 is the minimum possible token number, as everything below
// it is an ASCII value
for ($i = 256; $i < 1000; ++$i) {
// ASCII values map to themselves.
for ($i = 0; $i < 256; ++$i) {
$tokenMap[$i] = $i;
}
for (; $i < 1000; ++$i) {
if (\T_DOUBLE_COLON === $i) {
// T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
$tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;

26
lib/PhpParser/Lexer/Emulative.php
View File

@ -159,13 +159,7 @@ REGEX;
$pos = 0;
for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
$token = $this->tokens[$i];
if (\is_string($token)) {
// We assume that patches don't apply to string tokens
$pos += \strlen($token);
continue;
}
$len = \strlen($token[1]);
$len = \strlen($token->value);
$posDelta = 0;
while ($patchPos >= $pos && $patchPos < $pos + $len) {
$patchTextLen = \strlen($patchText);
@ -177,15 +171,15 @@ REGEX;
$c--;
} else {
// Remove from token string
$this->tokens[$i][1] = substr_replace(
$token[1], '', $patchPos - $pos + $posDelta, $patchTextLen
$this->tokens[$i]->value = substr_replace(
$token->value, '', $patchPos - $pos + $posDelta, $patchTextLen
);
$posDelta -= $patchTextLen;
}
} elseif ($patchType === 'add') {
// Insert into the token string
$this->tokens[$i][1] = substr_replace(
$token[1], $patchText, $patchPos - $pos + $posDelta, 0
$this->tokens[$i]->value = substr_replace(
$token->value, $patchText, $patchPos - $pos + $posDelta, 0
);
$posDelta += $patchTextLen;
} else {
@ -196,7 +190,7 @@ REGEX;
$patchIdx++;
if ($patchIdx >= \count($this->patches)) {
// No more patches, we're done
return;
break 2;
}
list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
@ -209,8 +203,12 @@ REGEX;
$pos += $len;
}
// A patch did not apply
assert(false);
// To retain a minimum amount of sanity, recompute token offsets in a separate loop...
$pos = 0;
foreach ($this->tokens as $token) {
$token->filePos = $pos;
$pos += \strlen($token->value);
}
}
/**

10
lib/PhpParser/Lexer/TokenEmulator/CoaleseEqualTokenEmulator.php
View File

@ -3,6 +3,7 @@
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\Lexer\Emulative;
use PhpParser\Token;
final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
{
@ -20,20 +21,17 @@ final class CoaleseEqualTokenEmulator implements TokenEmulatorInterface
{
// We need to manually iterate and manage a count because we'll change
// the tokens array on the way
$line = 1;
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
if (isset($tokens[$i + 1])) {
if ($tokens[$i][0] === T_COALESCE && $tokens[$i + 1] === '=') {
$token = $tokens[$i];
if ($token->id === \T_COALESCE && $tokens[$i + 1]->value === '=') {
array_splice($tokens, $i, 2, [
[Emulative::T_COALESCE_EQUAL, '??=', $line]
new Token(Emulative::T_COALESCE_EQUAL, '??=', $token->line, $token->filePos),
]);
$c--;
continue;
}
}
if (\is_array($tokens[$i])) {
$line += substr_count($tokens[$i][1], "\n");
}
}
return $tokens;

13
lib/PhpParser/Lexer/TokenEmulator/FnTokenEmulator.php
View File

@ -3,6 +3,7 @@
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\Lexer\Emulative;
use PhpParser\Token;
final class FnTokenEmulator implements TokenEmulatorInterface
{
@ -21,13 +22,13 @@ final class FnTokenEmulator implements TokenEmulatorInterface
// We need to manually iterate and manage a count because we'll change
// the tokens array on the way
foreach ($tokens as $i => $token) {
if ($token[0] === T_STRING && $token[1] === 'fn') {
if ($token->id === \T_STRING && $token->value === 'fn') {
$previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $i);
if ($previousNonSpaceToken !== null && $previousNonSpaceToken[0] === T_OBJECT_OPERATOR) {
if ($previousNonSpaceToken !== null && $previousNonSpaceToken->id === T_OBJECT_OPERATOR) {
continue;
}
$tokens[$i][0] = Emulative::T_FN;
$token->id = Emulative::T_FN;
}
}
@ -35,13 +36,13 @@ final class FnTokenEmulator implements TokenEmulatorInterface
}
/**
* @param mixed[] $tokens
* @return mixed[]|null
* @param Token[] $tokens
* @return Token|null
*/
private function getPreviousNonSpaceToken(array $tokens, int $start)
{
for ($i = $start - 1; $i >= 0; --$i) {
if ($tokens[$i][0] === T_WHITESPACE) {
if ($tokens[$i]->id === \T_WHITESPACE) {
continue;
}

25
lib/PhpParser/Lexer/TokenEmulator/NumericLiteralSeparatorEmulator.php
View File

@ -3,6 +3,7 @@
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\Lexer\Emulative;
use PhpParser\Token;
final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
{
@ -24,47 +25,46 @@ final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
return preg_match('~[0-9a-f]_[0-9a-f]~i', $code) !== false;
}
/**
* @param Token[] $tokens
* @return Token[]
*/
public function emulate(string $code, array $tokens): array
{
// We need to manually iterate and manage a count because we'll change
// the tokens array on the way
$codeOffset = 0;
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
$token = $tokens[$i];
$tokenLen = \strlen(\is_array($token) ? $token[1] : $token);
$tokenLen = \strlen($token->value);
if ($token[0] !== T_LNUMBER && $token[0] !== T_DNUMBER) {
$codeOffset += $tokenLen;
if ($token->id !== T_LNUMBER && $token->id !== T_DNUMBER) {
continue;
}
$res = preg_match(self::NUMBER, $code, $matches, 0, $codeOffset);
$res = preg_match(self::NUMBER, $code, $matches, 0, $token->filePos);
assert($res, "No number at number token position");
$match = $matches[0];
$matchLen = \strlen($match);
if ($matchLen === $tokenLen) {
// Original token already holds the full number.
$codeOffset += $tokenLen;
continue;
}
$tokenKind = $this->resolveIntegerOrFloatToken($match);
$newTokens = [[$tokenKind, $match, $token[2]]];
$newTokens = [new Token($tokenKind, $match, $token->line, $token->filePos)];
$numTokens = 1;
$len = $tokenLen;
while ($matchLen > $len) {
$nextToken = $tokens[$i + $numTokens];
$nextTokenText = \is_array($nextToken) ? $nextToken[1] : $nextToken;
$nextTokenLen = \strlen($nextTokenText);
$nextTokenLen = \strlen($nextToken->value);
$numTokens++;
if ($matchLen < $len + $nextTokenLen) {
// Split trailing characters into a partial token.
assert(is_array($nextToken), "Partial token should be an array token");
$partialText = substr($nextTokenText, $matchLen - $len);
$newTokens[] = [$nextToken[0], $partialText, $nextToken[2]];
$partialText = substr($nextToken->value, $matchLen - $len);
$newTokens[] = new Token($nextToken->id, $partialText, $nextToken->line, $nextToken->filePos);
break;
}
@ -73,7 +73,6 @@ final class NumericLiteralSeparatorEmulator implements TokenEmulatorInterface
array_splice($tokens, $i, $numTokens, $newTokens);
$c -= $numTokens - \count($newTokens);
$codeOffset += $matchLen;
}
return $tokens;

5
lib/PhpParser/Lexer/TokenEmulator/TokenEmulatorInterface.php
View File

@ -2,13 +2,16 @@
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\Token;
/** @internal */
interface TokenEmulatorInterface
{
public function isEmulationNeeded(string $code): bool;
/**
* @return array Modified Tokens
* @param Token[] $tokens
* @return Token[]
*/
public function emulate(string $code, array $tokens): array;
}

38
lib/PhpParser/PrettyPrinterAbstract.php
View File

@ -491,7 +491,7 @@ abstract class PrettyPrinterAbstract
$pos = 0;
$result = $this->pArray($stmts, $origStmts, $pos, 0, 'File', 'stmts', null);
if (null !== $result) {
$result .= $this->origTokens->getTokenCode($pos, count($origTokens), 0);
$result .= $this->origTokens->getTokenCode($pos, count($origTokens) - 1, 0);
} else {
// Fallback
// TODO Add <?php properly
@ -1213,8 +1213,8 @@ abstract class PrettyPrinterAbstract
$stripLeft = ['left' => \T_WHITESPACE];
$stripRight = ['right' => \T_WHITESPACE];
$stripDoubleArrow = ['right' => \T_DOUBLE_ARROW];
$stripColon = ['left' => ':'];
$stripEquals = ['left' => '='];
$stripColon = ['left' => \ord(':')];
$stripEquals = ['left' => \ord('=')];
$this->removalMap = [
'Expr_ArrayDimFetch->dim' => $stripBoth,
'Expr_ArrayItem->key' => $stripDoubleArrow,
@ -1254,22 +1254,22 @@ abstract class PrettyPrinterAbstract
// TODO: "yield" where both key and value are inserted doesn't work
// [$find, $beforeToken, $extraLeft, $extraRight]
$this->insertionMap = [
'Expr_ArrayDimFetch->dim' => ['[', false, null, null],
'Expr_ArrayDimFetch->dim' => [\ord('['), false, null, null],
'Expr_ArrayItem->key' => [null, false, null, ' => '],
'Expr_ArrowFunction->returnType' => [')', false, ' : ', null],
'Expr_Closure->returnType' => [')', false, ' : ', null],
'Expr_Ternary->if' => ['?', false, ' ', ' '],
'Expr_ArrowFunction->returnType' => [\ord(')'), false, ' : ', null],
'Expr_Closure->returnType' => [\ord(')'), false, ' : ', null],
'Expr_Ternary->if' => [\ord('?'), false, ' ', ' '],
'Expr_Yield->key' => [\T_YIELD, false, null, ' => '],
'Expr_Yield->value' => [\T_YIELD, false, ' ', null],
'Param->type' => [null, false, null, ' '],
'Param->default' => [null, false, ' = ', null],
'Stmt_Break->num' => [\T_BREAK, false, ' ', null],
'Stmt_ClassMethod->returnType' => [')', false, ' : ', null],
'Stmt_ClassMethod->returnType' => [\ord(')'), false, ' : ', null],
'Stmt_Class->extends' => [null, false, ' extends ', null],
'Expr_PrintableNewAnonClass->extends' => [null, ' extends ', null],
'Stmt_Continue->num' => [\T_CONTINUE, false, ' ', null],
'Stmt_Foreach->keyVar' => [\T_AS, false, null, ' => '],
'Stmt_Function->returnType' => [')', false, ' : ', null],
'Stmt_Function->returnType' => [\ord(')'), false, ' : ', null],
'Stmt_If->else' => [null, false, ' ', null],
'Stmt_Namespace->name' => [\T_NAMESPACE, false, ' ', null],
'Stmt_Property->type' => [\T_VARIABLE, true, null, ' '],
@ -1367,19 +1367,19 @@ abstract class PrettyPrinterAbstract
// [$find, $extraLeft, $extraRight]
$this->emptyListInsertionMap = [
'Expr_ArrowFunction->params' => ['(', '', ''],
'Expr_Closure->uses' => [')', ' use(', ')'],
'Expr_Closure->params' => ['(', '', ''],
'Expr_FuncCall->args' => ['(', '', ''],
'Expr_MethodCall->args' => ['(', '', ''],
'Expr_New->args' => ['(', '', ''],
'Expr_PrintableNewAnonClass->args' => ['(', '', ''],
'Expr_ArrowFunction->params' => [\ord('('), '', ''],
'Expr_Closure->uses' => [\ord(')'), ' use(', ')'],
'Expr_Closure->params' => [\ord('('), '', ''],
'Expr_FuncCall->args' => [\ord('('), '', ''],
'Expr_MethodCall->args' => [\ord('('), '', ''],
'Expr_New->args' => [\ord('('), '', ''],
'Expr_PrintableNewAnonClass->args' => [\ord('('), '', ''],
'Expr_PrintableNewAnonClass->implements' => [null, ' implements ', ''],
'Expr_StaticCall->args' => ['(', '', ''],
'Expr_StaticCall->args' => [\ord('('), '', ''],
'Stmt_Class->implements' => [null, ' implements ', ''],
'Stmt_ClassMethod->params' => ['(', '', ''],
'Stmt_ClassMethod->params' => [\ord('('), '', ''],
'Stmt_Interface->extends' => [null, ' extends ', ''],
'Stmt_Function->params' => ['(', '', ''],
'Stmt_Function->params' => [\ord('('), '', ''],
/* These cannot be empty to start with:
* Expr_Isset->vars

21
lib/PhpParser/Token.php Normal file
View File

@ -0,0 +1,21 @@
<?php declare(strict_types=1);
namespace PhpParser;
class Token {
/** @var int Token id (T_* constant) */
public $id;
/** @var string Textual value of the token */
public $value;
/** @var int Start line number of the token */
public $line;
/** @var int Offset of the token in the source code */
public $filePos;
public function __construct(int $id, string $value, int $line, int $filePos) {
$this->id = $id;
$this->value = $value;
$this->line = $line;
$this->filePos = $filePos;
}
}

17
test/PhpParser/LexerTest.php
View File

@ -247,17 +247,18 @@ class LexerTest extends \PHPUnit\Framework\TestCase
public function testGetTokens() {
$code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
$expectedTokens = [
[T_OPEN_TAG, '<?php ', 1],
[T_CONSTANT_ENCAPSED_STRING, '"a"', 1],
';',
[T_WHITESPACE, "\n", 1],
[T_COMMENT, '// foo' . "\n", 2],
[T_CONSTANT_ENCAPSED_STRING, '"b"', 3],
';',
new Token(T_OPEN_TAG, '<?php ', 1, 0),
new Token(T_CONSTANT_ENCAPSED_STRING, '"a"', 1, 6),
new Token(\ord(';'), ';', 1, 9),
new Token(T_WHITESPACE, "\n", 1, 10),
new Token(T_COMMENT, '// foo' . "\n", 2, 11),
new Token(T_CONSTANT_ENCAPSED_STRING, '"b"', 3, 18),
new Token(\ord(';'), ';', 3, 21),
new Token(0, "\0", 3, 22),
];
$lexer = $this->getLexer();
$lexer->startLexing($code);
$this->assertSame($expectedTokens, $lexer->getTokens());
$this->assertEquals($expectedTokens, $lexer->getTokens());
}
}