diff --git a/grammar/tokens.y b/grammar/tokens.y index 744781c..b0b0360 100644 --- a/grammar/tokens.y +++ b/grammar/tokens.y @@ -109,4 +109,5 @@ %token T_ELLIPSIS %token T_NAME_FULLY_QUALIFIED %token T_NAME_QUALIFIED -%token T_NAME_RELATIVE \ No newline at end of file +%token T_NAME_RELATIVE +%token T_ATTRIBUTE \ No newline at end of file diff --git a/lib/PhpParser/Lexer.php b/lib/PhpParser/Lexer.php index 9873ae2..738dff3 100644 --- a/lib/PhpParser/Lexer.php +++ b/lib/PhpParser/Lexer.php @@ -421,6 +421,7 @@ class Lexer 'T_NAME_RELATIVE', 'T_MATCH', 'T_NULLSAFE_OBJECT_OPERATOR', + 'T_ATTRIBUTE', ]; // PHP-Parser might be used together with another library that also emulates some or all @@ -510,6 +511,7 @@ class Lexer $tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE; $tokenMap[\T_MATCH] = Tokens::T_MATCH; $tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR; + $tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE; return $tokenMap; } diff --git a/lib/PhpParser/Lexer/Emulative.php b/lib/PhpParser/Lexer/Emulative.php index 7698abb..cf1f8e5 100644 --- a/lib/PhpParser/Lexer/Emulative.php +++ b/lib/PhpParser/Lexer/Emulative.php @@ -5,6 +5,7 @@ namespace PhpParser\Lexer; use PhpParser\Error; use PhpParser\ErrorHandler; use PhpParser\Lexer; +use PhpParser\Lexer\TokenEmulator\AttributeEmulator; use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator; use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator; use PhpParser\Lexer\TokenEmulator\FnTokenEmulator; @@ -49,6 +50,7 @@ class Emulative extends Lexer new CoaleseEqualTokenEmulator(), new NumericLiteralSeparatorEmulator(), new NullsafeTokenEmulator(), + new AttributeEmulator(), ]; // Collect emulators that are relevant for the PHP version we're running @@ -81,6 +83,7 @@ class Emulative extends Lexer $collector = new ErrorHandler\Collecting(); parent::startLexing($code, $collector); + $this->sortPatches(); $this->fixupTokens(); $errors = $collector->getErrors(); @@ -106,6 +109,15 @@ class Emulative extends Lexer && version_compare($this->targetPhpVersion, $emulatorPhpVersion, '<'); } + private function sortPatches() + { + // Patches may be contributed by different emulators. + // Make sure they are sorted by increasing patch position. + usort($this->patches, function($p1, $p2) { + return $p1[0] <=> $p2[0]; + }); + } + private function fixupTokens() { if (\count($this->patches) === 0) { @@ -122,7 +134,20 @@ class Emulative extends Lexer for ($i = 0, $c = \count($this->tokens); $i < $c; $i++) { $token = $this->tokens[$i]; if (\is_string($token)) { - // We assume that patches don't apply to string tokens + if ($patchPos === $pos) { + // Only support replacement for string tokens. + assert($patchType === 'replace'); + $this->tokens[$i] = $patchText; + + // Fetch the next patch + $patchIdx++; + if ($patchIdx >= \count($this->patches)) { + // No more patches, we're done + return; + } + list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx]; + } + $pos += \strlen($token); continue; } @@ -150,6 +175,11 @@ class Emulative extends Lexer $token[1], $patchText, $patchPos - $pos + $posDelta, 0 ); $posDelta += $patchTextLen; + } else if ($patchType === 'replace') { + // Replace inside the token string + $this->tokens[$i][1] = substr_replace( + $token[1], $patchText, $patchPos - $pos + $posDelta, $patchTextLen + ); } else { assert(false); } @@ -196,7 +226,7 @@ class Emulative extends Lexer if ($patchType === 'add') { $posDelta += strlen($patchText); $lineDelta += substr_count($patchText, "\n"); - } else { + } else if ($patchType === 'remove') { $posDelta -= strlen($patchText); $lineDelta -= substr_count($patchText, "\n"); } diff --git a/lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php b/lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php new file mode 100644 index 0000000..6776a51 --- /dev/null +++ b/lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php @@ -0,0 +1,56 @@ +', [ [Tokens::T_NULLSAFE_OBJECT_OPERATOR, '?->'], ]], + ['#[Attr]', [ + [Tokens::T_ATTRIBUTE, '#['], + [Tokens::T_STRING, 'Attr'], + [ord(']'), ']'], + ]], + ["#[\nAttr\n]", [ + [Tokens::T_ATTRIBUTE, '#['], + [Tokens::T_STRING, 'Attr'], + [ord(']'), ']'], + ]], + // Test interaction of two patch-based emulators + ["<<