Make canonicalization less aggressive

test/PhpParser/CodeTestAbstract.php

@@ -13,6 +13,7 @@ abstract class CodeTestAbstract extends \PHPUnit_Framework_TestCase
         foreach ($it as $file) {
             $fileName = realpath($file->getPathname());
             $fileContents = file_get_contents($fileName);
+            $fileContents = canonicalize($fileContents);
 
             // evaluate @@{expr}@@ expressions
             $fileContents = preg_replace_callback(
@@ -24,7 +25,7 @@ abstract class CodeTestAbstract extends \PHPUnit_Framework_TestCase
             );
 
             // parse sections
-            $parts = array_map('trim', explode('-----', $fileContents));
+            $parts = explode("\n-----\n", $fileContents);
 
             // first part is the name
             $name = array_shift($parts) . ' (' . $fileName . ')';
@@ -34,7 +35,7 @@ abstract class CodeTestAbstract extends \PHPUnit_Framework_TestCase
             $chunks = array_chunk($parts, 2);
             foreach ($chunks as $i => $chunk) {
                 $dataSetName = $shortName . (count($chunks) > 1 ? '#' . $i : '');
-                list($expected, $mode) = $this->extractMode(canonicalize($chunk[1]));
+                list($expected, $mode) = $this->extractMode($chunk[1]);
                 $tests[$dataSetName] = array($name, $chunk[0], $expected, $mode);
             }
         }

test/bootstrap.php

@@ -5,12 +5,16 @@ namespace PhpParser;
 require __DIR__ . '/../vendor/autoload.php';
 
 function canonicalize($str) {
-    // trim from both sides
-    $str = trim($str);
+    // normalize EOL style
+    $str = str_replace("\r\n", "\n", $str);
 
-    // normalize EOL to \n
-    $str = str_replace(array("\r\n", "\r"), "\n", $str);
+    // trim newlines at end
+    $str = rtrim($str, "\n");
 
-    // trim right side of all lines
-    return implode("\n", array_map('rtrim', explode("\n", $str)));
+    // remove trailing whitespace on all lines
+    $lines = explode("\n", $str);
+    $lines = array_map(function($line) {
+        return rtrim($line, " \t");
+    }, $lines);
+    return implode("\n", $lines);
 }

test/code/parser/scalar/constantString.test

@@ -52,8 +52,7 @@ array(
     )
     11: Scalar_String(
         value: !"!\!$!
-    !
-!@@{ "\t" }@@!@@{ "\f" }@@!@@{ "\v" }@@!@@{ chr(27) /* "\e" */ }@@!\a
+    !@@{ "\r" }@@!@@{ "\t" }@@!@@{ "\f" }@@!@@{ "\v" }@@!@@{ chr(27) /* "\e" */ }@@!\a
     )
     12: Scalar_String(
         value: !@@{ chr(255) }@@!@@{ chr(255) }@@!@@{ chr(0) }@@!@@{ chr(0) }@@!

test/code/parser/scalar/docString.test

@@ -28,7 +28,6 @@ b<<<EOS
 Binary
 EOS;
 
-// comment to force line break before EOF
 -----
 array(
     0: Scalar_String(
@@ -88,9 +87,4 @@ array(
     6: Scalar_String(
         value: Binary
     )
-    7: Stmt_Nop(
-        comments: array(
-            0: // comment to force line break before EOF
-        )
-    )
 )

test/code/parser/scalar/docStringNewlines.test

@@ -12,7 +12,6 @@ Trailing newlines in doc strings
 <<<'EOF'@@{ "\r\nFoo\r\n\r\n" }@@EOF;
 <<<EOF@@{ "\r\n\$var\r\n\r\n" }@@EOF;
 
-// comment to force line break before EOF
 -----
 array(
     0: Scalar_String(
@@ -59,9 +58,4 @@ array(
             )
         )
     )
-    8: Stmt_Nop(
-        comments: array(
-            0: // comment to force line break before EOF
-        )
-    )
 )

test/code/prettyPrinter/onlyPHP.file-test

@@ -11,8 +11,6 @@ echo 'Foo Bar';
 echo 'Bar Foo';
 -----
 <?php
-// avoid trim
+
 -----
 <?php
-
-// avoid trim