danog/PHP-Parser
1
0
Fork 0
mirror of https://github.com/danog/PHP-Parser.git synced 2024-11-30 04:19:30 +01:00
Code Issues Projects Releases Wiki Activity

Emulate PHP 8 attribute syntax

Browse Source 
Perform emulation by replacing #[ with %[, then patching % back
to # and coalescing #[ into T_ATTRIBUTE if it is a freestanding
token.
This commit is contained in:
Nikita Popov 2020-09-06 17:04:33 +02:00
parent 75abbbd2d4
commit f66a32e2df
8 changed files with 182 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
grammar/tokens.y
View File

@ -109,4 +109,5 @@
%token T_ELLIPSIS
%token T_NAME_FULLY_QUALIFIED
%token T_NAME_QUALIFIED
%token T_NAME_RELATIVE
%token T_NAME_RELATIVE
%token T_ATTRIBUTE

2
lib/PhpParser/Lexer.php
View File

@ -421,6 +421,7 @@ class Lexer
'T_NAME_RELATIVE',
'T_MATCH',
'T_NULLSAFE_OBJECT_OPERATOR',
'T_ATTRIBUTE',
];
// PHP-Parser might be used together with another library that also emulates some or all
@ -510,6 +511,7 @@ class Lexer
$tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
$tokenMap[\T_MATCH] = Tokens::T_MATCH;
$tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
$tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
return $tokenMap;
}

34
lib/PhpParser/Lexer/Emulative.php
View File

@ -5,6 +5,7 @@ namespace PhpParser\Lexer;
use PhpParser\Error;
use PhpParser\ErrorHandler;
use PhpParser\Lexer;
use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
@ -49,6 +50,7 @@ class Emulative extends Lexer
new CoaleseEqualTokenEmulator(),
new NumericLiteralSeparatorEmulator(),
new NullsafeTokenEmulator(),
new AttributeEmulator(),
];
// Collect emulators that are relevant for the PHP version we're running
@ -81,6 +83,7 @@ class Emulative extends Lexer
$collector = new ErrorHandler\Collecting();
parent::startLexing($code, $collector);
$this->sortPatches();
$this->fixupTokens();
$errors = $collector->getErrors();
@ -106,6 +109,15 @@ class Emulative extends Lexer
&& version_compare($this->targetPhpVersion, $emulatorPhpVersion, '<');
}
private function sortPatches()
{
// Patches may be contributed by different emulators.
// Make sure they are sorted by increasing patch position.
usort($this->patches, function($p1, $p2) {
return $p1[0] <=> $p2[0];
});
}
private function fixupTokens()
{
if (\count($this->patches) === 0) {
@ -122,7 +134,20 @@ class Emulative extends Lexer
for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) {
$token = $this->tokens[$i];
if (\is_string($token)) {
// We assume that patches don't apply to string tokens
if ($patchPos === $pos) {
// Only support replacement for string tokens.
assert($patchType === 'replace');
$this->tokens[$i] = $patchText;
// Fetch the next patch
$patchIdx++;
if ($patchIdx >= \count($this->patches)) {
// No more patches, we're done
return;
}
list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
}
$pos += \strlen($token);
continue;
}
@ -150,6 +175,11 @@ class Emulative extends Lexer
$token[1], $patchText, $patchPos - $pos + $posDelta, 0
);
$posDelta += $patchTextLen;
} else if ($patchType === 'replace') {
// Replace inside the token string
$this->tokens[$i][1] = substr_replace(
$token[1], $patchText, $patchPos - $pos + $posDelta, $patchTextLen
);
} else {
assert(false);
}
@ -196,7 +226,7 @@ class Emulative extends Lexer
if ($patchType === 'add') {
$posDelta += strlen($patchText);
$lineDelta += substr_count($patchText, "\n");
} else {
} else if ($patchType === 'remove') {
$posDelta -= strlen($patchText);
$lineDelta -= substr_count($patchText, "\n");
}

56
lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php Normal file
View File

@ -0,0 +1,56 @@
<?php declare(strict_types=1);
namespace PhpParser\Lexer\TokenEmulator;
use PhpParser\Lexer\Emulative;
final class AttributeEmulator extends TokenEmulator
{
public function getPhpVersion(): string
{
return Emulative::PHP_8_0;
}
public function isEmulationNeeded(string $code) : bool
{
return strpos($code, '#[') !== false;
}
public function emulate(string $code, array $tokens): array
{
// We need to manually iterate and manage a count because we'll change
// the tokens array on the way.
$line = 1;
for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
if ($tokens[$i] === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1] === '[') {
array_splice($tokens, $i, 2, [
[\T_ATTRIBUTE, '#[', $line]
]);
$c--;
continue;
}
if (\is_array($tokens[$i])) {
$line += substr_count($tokens[$i][1], "\n");
}
}
return $tokens;
}
public function reverseEmulate(string $code, array $tokens): array
{
// TODO
return $tokens;
}
public function preprocessCode(string $code, array &$patches): string {
$pos = 0;
while (false !== $pos = strpos($code, '#[', $pos)) {
// Replace #[ with %[
$code[$pos] = '%';
$patches[] = [$pos, 'replace', '#'];
$pos += 2;
}
return $code;
}
}

61
lib/PhpParser/Parser/Php5.php
View File

@ -17,11 +17,11 @@ use PhpParser\Node\Stmt;
*/
class Php5 extends \PhpParser\ParserAbstract
{
protected $tokenToSymbolMapSize = 391;
protected $tokenToSymbolMapSize = 392;
protected $actionTableSize = 1069;
protected $gotoTableSize = 580;
protected $invalidSymbol = 164;
protected $invalidSymbol = 165;
protected $errorSymbol = 1;
protected $defaultAction = -32766;
protected $unexpectedTokenRule = 32767;
@ -193,36 +193,37 @@ class Php5 extends \PhpParser\ParserAbstract
"'`'",
"']'",
"'\"'",
"T_NULLSAFE_OBJECT_OPERATOR"
"T_NULLSAFE_OBJECT_OPERATOR",
"T_ATTRIBUTE"
);
protected $tokenToSymbol = array(
0, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 55, 162, 164, 159, 54, 37, 164,
157, 158, 52, 49, 8, 50, 51, 53, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 31, 154,
43, 16, 45, 30, 67, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 69, 164, 161, 36, 164, 160, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 155, 35, 156, 57, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 1, 2, 3, 4,
0, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 55, 162, 165, 159, 54, 37, 165,
157, 158, 52, 49, 8, 50, 51, 53, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 31, 154,
43, 16, 45, 30, 67, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 69, 165, 161, 36, 165, 160, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 155, 35, 156, 57, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 1, 2, 3, 4,
5, 6, 7, 9, 10, 11, 12, 13, 14, 15,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 32, 33, 34, 38, 39, 40, 41,
@ -236,7 +237,7 @@ class Php5 extends \PhpParser\ParserAbstract
124, 125, 126, 127, 128, 129, 130, 131, 163, 132,
133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
143, 144, 145, 146, 147, 148, 149, 150, 151, 152,
153
153, 164
);
protected $action = array(

61
lib/PhpParser/Parser/Php7.php
View File

@ -17,11 +17,11 @@ use PhpParser\Node\Stmt;
*/
class Php7 extends \PhpParser\ParserAbstract
{
protected $tokenToSymbolMapSize = 391;
protected $tokenToSymbolMapSize = 392;
protected $actionTableSize = 1124;
protected $gotoTableSize = 498;
protected $invalidSymbol = 164;
protected $invalidSymbol = 165;
protected $errorSymbol = 1;
protected $defaultAction = -32766;
protected $unexpectedTokenRule = 32767;
@ -193,36 +193,37 @@ class Php7 extends \PhpParser\ParserAbstract
"'`'",
"']'",
"'\"'",
"'$'"
"'$'",
"T_ATTRIBUTE"
);
protected $tokenToSymbol = array(
0, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 55, 162, 164, 163, 54, 37, 164,
158, 159, 52, 49, 8, 50, 51, 53, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 31, 155,
43, 16, 45, 30, 67, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 69, 164, 161, 36, 164, 160, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 156, 35, 157, 57, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 164, 164, 164, 164,
164, 164, 164, 164, 164, 164, 1, 2, 3, 4,
0, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 55, 162, 165, 163, 54, 37, 165,
158, 159, 52, 49, 8, 50, 51, 53, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 31, 155,
43, 16, 45, 30, 67, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 69, 165, 161, 36, 165, 160, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 156, 35, 157, 57, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 165, 165, 165, 165,
165, 165, 165, 165, 165, 165, 1, 2, 3, 4,
5, 6, 7, 9, 10, 11, 12, 13, 14, 15,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 32, 33, 34, 38, 39, 40, 41,
@ -236,7 +237,7 @@ class Php7 extends \PhpParser\ParserAbstract
124, 125, 126, 127, 128, 129, 130, 131, 132, 133,
134, 135, 136, 137, 138, 139, 140, 141, 142, 143,
144, 145, 146, 147, 148, 149, 150, 151, 152, 153,
154
154, 164
);
protected $action = array(

1
lib/PhpParser/Parser/Tokens.php
View File

@ -140,4 +140,5 @@ final class Tokens
const T_NAME_FULLY_QUALIFIED = 388;
const T_NAME_QUALIFIED = 389;
const T_NAME_RELATIVE = 390;
const T_ATTRIBUTE = 391;
}

27
test/PhpParser/Lexer/EmulativeTest.php
View File

@ -275,6 +275,33 @@ class EmulativeTest extends LexerTest
['?->', [
[Tokens::T_NULLSAFE_OBJECT_OPERATOR, '?->'],
]],
['#[Attr]', [
[Tokens::T_ATTRIBUTE, '#['],
[Tokens::T_STRING, 'Attr'],
[ord(']'), ']'],
]],
["#[\nAttr\n]", [
[Tokens::T_ATTRIBUTE, '#['],
[Tokens::T_STRING, 'Attr'],
[ord(']'), ']'],
]],
// Test interaction of two patch-based emulators
["<<<LABEL\n LABEL, #[Attr]", [
[Tokens::T_START_HEREDOC, "<<<LABEL\n"],
[Tokens::T_END_HEREDOC, " LABEL"],
[ord(','), ','],
[Tokens::T_ATTRIBUTE, '#['],
[Tokens::T_STRING, 'Attr'],
[ord(']'), ']'],
]],
["#[Attr] <<<LABEL\n LABEL,", [
[Tokens::T_ATTRIBUTE, '#['],
[Tokens::T_STRING, 'Attr'],
[ord(']'), ']'],
[Tokens::T_START_HEREDOC, "<<<LABEL\n"],
[Tokens::T_END_HEREDOC, " LABEL"],
[ord(','), ','],
]],
];
}