danog/PHP-Parser
1
0
Fork 0
mirror of https://github.com/danog/PHP-Parser.git synced 2024-11-30 04:19:30 +01:00
Code Issues Projects Releases Wiki Activity

Fix compatibility with T_BAD_CHARACTER in PHP 7.4

Browse Source
This commit is contained in:
Nikita Popov 2019-08-12 22:10:02 +02:00
parent a1f72690ef
commit 4fd36b9946
1 changed files with 14 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/PhpParser/Lexer.php
View File

@ -6,11 +6,6 @@ use PhpParser\Parser\Tokens;
class Lexer class Lexer
{ {
/* Token ID used for illegal characters part of the token stream. These are dropped by token_get_all(),
* but we restore them here to make sure that the tokens cover the full original text, and to prevent
* file positions from going out of sync. */
const T_BAD_CHARACTER = -1;
protected $code; protected $code;
protected $tokens; protected $tokens;
protected $pos; protected $pos;
@ -42,10 +37,15 @@ class Lexer
// map from internal tokens to PhpParser tokens // map from internal tokens to PhpParser tokens
$this->tokenMap = $this->createTokenMap(); $this->tokenMap = $this->createTokenMap();
// Compatibility define for PHP < 7.4
if (!defined('T_BAD_CHARACTER')) {
\define('T_BAD_CHARACTER', -1);
}
// map of tokens to drop while lexing (the map is only used for isset lookup, // map of tokens to drop while lexing (the map is only used for isset lookup,
// that's why the value is simply set to 1; the value is never actually used.) // that's why the value is simply set to 1; the value is never actually used.)
$this->dropTokens = array_fill_keys( $this->dropTokens = array_fill_keys(
[\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, self::T_BAD_CHARACTER], 1 [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
); );
$defaultAttributes = ['comments', 'startLine', 'endLine']; $defaultAttributes = ['comments', 'startLine', 'endLine'];
@ -109,7 +109,7 @@ class Lexer
); );
} }
$tokens[] = [self::T_BAD_CHARACTER, $chr, $line]; $tokens[] = [\T_BAD_CHARACTER, $chr, $line];
$errorHandler->handleError(new Error($errorMsg, [ $errorHandler->handleError(new Error($errorMsg, [
'startLine' => $line, 'startLine' => $line,
'endLine' => $line, 'endLine' => $line,
@ -161,6 +161,13 @@ class Lexer
$numTokens = \count($this->tokens); $numTokens = \count($this->tokens);
for ($i = 0; $i < $numTokens; $i++) { for ($i = 0; $i < $numTokens; $i++) {
$token = $this->tokens[$i]; $token = $this->tokens[$i];
// Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
// In this case we only need to emit an error.
if ($token[0] === \T_BAD_CHARACTER) {
$this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
}
$tokenValue = \is_string($token) ? $token : $token[1]; $tokenValue = \is_string($token) ? $token : $token[1];
$tokenLen = \strlen($tokenValue); $tokenLen = \strlen($tokenValue);