From 4fd36b99463b08767ac470bf475cec667d727c86 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikita.ppv@gmail.com>
Date: Mon, 12 Aug 2019 22:10:02 +0200
Subject: [PATCH] Fix compatibility with T_BAD_CHARACTER in PHP 7.4

---
 lib/PhpParser/Lexer.php | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/lib/PhpParser/Lexer.php b/lib/PhpParser/Lexer.php
index 51ffd1e..07ec5ca 100644
--- a/lib/PhpParser/Lexer.php
+++ b/lib/PhpParser/Lexer.php
@@ -6,11 +6,6 @@ use PhpParser\Parser\Tokens;
 
 class Lexer
 {
-    /* Token ID used for illegal characters part of the token stream. These are dropped by token_get_all(),
-     * but we restore them here to make sure that the tokens cover the full original text, and to prevent
-     * file positions from going out of sync. */
-    const T_BAD_CHARACTER = -1;
-
     protected $code;
     protected $tokens;
     protected $pos;
@@ -42,10 +37,15 @@ class Lexer
         // map from internal tokens to PhpParser tokens
         $this->tokenMap = $this->createTokenMap();
 
+        // Compatibility define for PHP < 7.4
+        if (!defined('T_BAD_CHARACTER')) {
+            \define('T_BAD_CHARACTER', -1);
+        }
+
         // map of tokens to drop while lexing (the map is only used for isset lookup,
         // that's why the value is simply set to 1; the value is never actually used.)
         $this->dropTokens = array_fill_keys(
-            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, self::T_BAD_CHARACTER], 1
+            [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
         );
 
         $defaultAttributes = ['comments', 'startLine', 'endLine'];
@@ -109,7 +109,7 @@ class Lexer
                 );
             }
 
-            $tokens[] = [self::T_BAD_CHARACTER, $chr, $line];
+            $tokens[] = [\T_BAD_CHARACTER, $chr, $line];
             $errorHandler->handleError(new Error($errorMsg, [
                 'startLine' => $line,
                 'endLine' => $line,
@@ -161,6 +161,13 @@ class Lexer
         $numTokens = \count($this->tokens);
         for ($i = 0; $i < $numTokens; $i++) {
             $token = $this->tokens[$i];
+
+            // Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
+            // In this case we only need to emit an error.
+            if ($token[0] === \T_BAD_CHARACTER) {
+                $this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
+            }
+
             $tokenValue = \is_string($token) ? $token : $token[1];
             $tokenLen = \strlen($tokenValue);