Support file offsets in emulative lexer as well

Also run normal lexer tests against emulative lexer and fix a bug
in __halt_compiler() handling found as a result.
This commit is contained in:
nikic 2014-11-27 20:57:38 +01:00
parent d774dbc1b7
commit 7c98ad6f9b
3 changed files with 38 additions and 27 deletions

View File

@ -20,8 +20,8 @@ class Emulative extends \PhpParser\Lexer
const PHP_5_5 = '5.5.0beta1'; const PHP_5_5 = '5.5.0beta1';
const PHP_5_4 = '5.4.0beta1'; const PHP_5_4 = '5.4.0beta1';
public function __construct() { public function __construct(array $options = array()) {
parent::__construct(); parent::__construct($options);
$newKeywordsPerVersion = array( $newKeywordsPerVersion = array(
self::PHP_5_5 => array( self::PHP_5_5 => array(
@ -60,6 +60,10 @@ class Emulative extends \PhpParser\Lexer
if ($preprocessedCode !== $code) { if ($preprocessedCode !== $code) {
$this->postprocessTokens(); $this->postprocessTokens();
} }
// Set code property back to the original code, so __halt_compiler()
// handling and (start|end)FilePos attributes use the correct offsets
$this->code = $code;
} }
/* /*
@ -179,4 +183,4 @@ class Emulative extends \PhpParser\Lexer
return $token; return $token;
} }
} }

View File

@ -2,36 +2,36 @@
namespace PhpParser\Lexer; namespace PhpParser\Lexer;
use PhpParser\LexerTest;
use PhpParser\Parser; use PhpParser\Parser;
class EmulativeTest extends \PHPUnit_Framework_TestCase class EmulativeTest extends LexerTest
{ {
/** @var Emulative */ protected function getLexer(array $options = array()) {
protected $lexer; return new Emulative($options);
protected function setUp() {
$this->lexer = new Emulative;
} }
/** /**
* @dataProvider provideTestReplaceKeywords * @dataProvider provideTestReplaceKeywords
*/ */
public function testReplaceKeywords($keyword, $expectedToken) { public function testReplaceKeywords($keyword, $expectedToken) {
$this->lexer->startLexing('<?php ' . $keyword); $lexer = $this->getLexer();
$lexer->startLexing('<?php ' . $keyword);
$this->assertSame($expectedToken, $this->lexer->getNextToken()); $this->assertSame($expectedToken, $lexer->getNextToken());
$this->assertSame(0, $this->lexer->getNextToken()); $this->assertSame(0, $lexer->getNextToken());
} }
/** /**
* @dataProvider provideTestReplaceKeywords * @dataProvider provideTestReplaceKeywords
*/ */
public function testNoReplaceKeywordsAfterObjectOperator($keyword) { public function testNoReplaceKeywordsAfterObjectOperator($keyword) {
$this->lexer->startLexing('<?php ->' . $keyword); $lexer = $this->getLexer();
$lexer->startLexing('<?php ->' . $keyword);
$this->assertSame(Parser::T_OBJECT_OPERATOR, $this->lexer->getNextToken()); $this->assertSame(Parser::T_OBJECT_OPERATOR, $lexer->getNextToken());
$this->assertSame(Parser::T_STRING, $this->lexer->getNextToken()); $this->assertSame(Parser::T_STRING, $lexer->getNextToken());
$this->assertSame(0, $this->lexer->getNextToken()); $this->assertSame(0, $lexer->getNextToken());
} }
public function provideTestReplaceKeywords() { public function provideTestReplaceKeywords() {
@ -58,14 +58,15 @@ class EmulativeTest extends \PHPUnit_Framework_TestCase
* @dataProvider provideTestLexNewFeatures * @dataProvider provideTestLexNewFeatures
*/ */
public function testLexNewFeatures($code, array $expectedTokens) { public function testLexNewFeatures($code, array $expectedTokens) {
$this->lexer->startLexing('<?php ' . $code); $lexer = $this->getLexer();
$lexer->startLexing('<?php ' . $code);
foreach ($expectedTokens as $expectedToken) { foreach ($expectedTokens as $expectedToken) {
list($expectedTokenType, $expectedTokenText) = $expectedToken; list($expectedTokenType, $expectedTokenText) = $expectedToken;
$this->assertSame($expectedTokenType, $this->lexer->getNextToken($text)); $this->assertSame($expectedTokenType, $lexer->getNextToken($text));
$this->assertSame($expectedTokenText, $text); $this->assertSame($expectedTokenText, $text);
} }
$this->assertSame(0, $this->lexer->getNextToken()); $this->assertSame(0, $lexer->getNextToken());
} }
/** /**
@ -73,11 +74,13 @@ class EmulativeTest extends \PHPUnit_Framework_TestCase
*/ */
public function testLeaveStuffAloneInStrings($code) { public function testLeaveStuffAloneInStrings($code) {
$stringifiedToken = '"' . addcslashes($code, '"\\') . '"'; $stringifiedToken = '"' . addcslashes($code, '"\\') . '"';
$this->lexer->startLexing('<?php ' . $stringifiedToken);
$this->assertSame(Parser::T_CONSTANT_ENCAPSED_STRING, $this->lexer->getNextToken($text)); $lexer = $this->getLexer();
$lexer->startLexing('<?php ' . $stringifiedToken);
$this->assertSame(Parser::T_CONSTANT_ENCAPSED_STRING, $lexer->getNextToken($text));
$this->assertSame($stringifiedToken, $text); $this->assertSame($stringifiedToken, $text);
$this->assertSame(0, $this->lexer->getNextToken()); $this->assertSame(0, $lexer->getNextToken());
} }
public function provideTestLexNewFeatures() { public function provideTestLexNewFeatures() {
@ -113,4 +116,4 @@ class EmulativeTest extends \PHPUnit_Framework_TestCase
)), )),
); );
} }
} }

View File

@ -4,12 +4,16 @@ namespace PhpParser;
class LexerTest extends \PHPUnit_Framework_TestCase class LexerTest extends \PHPUnit_Framework_TestCase
{ {
/* To allow overwriting in parent class */
protected function getLexer(array $options = array()) {
return new Lexer($options);
}
/** /**
* @dataProvider provideTestError * @dataProvider provideTestError
*/ */
public function testError($code, $message) { public function testError($code, $message) {
$lexer = new Lexer; $lexer = $this->getLexer();
try { try {
$lexer->startLexing($code); $lexer->startLexing($code);
} catch (Error $e) { } catch (Error $e) {
@ -33,7 +37,7 @@ class LexerTest extends \PHPUnit_Framework_TestCase
* @dataProvider provideTestLex * @dataProvider provideTestLex
*/ */
public function testLex($code, $options, $tokens) { public function testLex($code, $options, $tokens) {
$lexer = new Lexer($options); $lexer = $this->getLexer($options);
$lexer->startLexing($code); $lexer->startLexing($code);
while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) { while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
$token = array_shift($tokens); $token = array_shift($tokens);
@ -165,12 +169,12 @@ class LexerTest extends \PHPUnit_Framework_TestCase
* @dataProvider provideTestHaltCompiler * @dataProvider provideTestHaltCompiler
*/ */
public function testHandleHaltCompiler($code, $remaining) { public function testHandleHaltCompiler($code, $remaining) {
$lexer = new Lexer; $lexer = $this->getLexer();
$lexer->startLexing($code); $lexer->startLexing($code);
while (Parser::T_HALT_COMPILER !== $lexer->getNextToken()); while (Parser::T_HALT_COMPILER !== $lexer->getNextToken());
$this->assertSame($lexer->handleHaltCompiler(), $remaining); $this->assertSame($remaining, $lexer->handleHaltCompiler());
$this->assertSame(0, $lexer->getNextToken()); $this->assertSame(0, $lexer->getNextToken());
} }