Collect normal comments too, not only doc comments

Comments and doc comments are now saved in the 'comments' attribute, as an
array. The are instances of PHPParser_Comment[_Doc].

lib/PHPParser/Comment.php

@@ -0,0 +1,42 @@
+<?php
+
+class PHPParser_Comment
+{
+    protected $text;
+
+    /**
+     * Constructs a comment node.
+     *
+     * @param string $text Comment text (including comment delimiters like /*)
+     */
+    public function __construct($text) {
+        $this->text = $text;
+    }
+
+    /**
+     * Gets the comment text.
+     *
+     * @return string The comment text (including comment delimiters like /*)
+     */
+    public function getText() {
+        return $this->text;
+    }
+
+    /**
+     * Sets the comment text.
+     *
+     * @param string $text The comment text (including comment delimiters like /*)
+     */
+    public function setText($text) {
+        $this->text = $text;
+    }
+
+    /**
+     * Gets the comment text.
+     *
+     * @return string The comment text (including comment delimiters like /*)
+     */
+    public function __toString() {
+        return $this->text;
+    }
+}

lib/PHPParser/Comment/Doc.php

@@ -0,0 +1,5 @@
+<?php
+
+class PHPParser_Comment_Doc extends PHPParser_Comment
+{
+}

lib/PHPParser/Lexer.php

@@ -19,7 +19,7 @@ class PHPParser_Lexer
 
         // map of tokens to drop while lexing (the map is only used for isset lookup,
         // that's why the value is simply set to 1; the value is never actually used.)
-        $this->dropTokens = array_fill_keys(array(T_WHITESPACE, T_COMMENT, T_OPEN_TAG), 1);
+        $this->dropTokens = array_fill_keys(array(T_WHITESPACE, T_OPEN_TAG), 1);
     }
 
     /**
@@ -101,8 +101,10 @@ class PHPParser_Lexer
             } else {
                 $this->line += substr_count($token[1], "\n");
 
-                if (T_DOC_COMMENT === $token[0]) {
+                if (T_COMMENT === $token[0]) {
-                    $startAttributes['docComment'] = $token[1];
+                    $startAttributes['comments'][] = new PHPParser_Comment($token[1]);
+                } elseif (T_DOC_COMMENT === $token[0]) {
+                    $startAttributes['comments'][] = new PHPParser_Comment_Doc($token[1]);
                 } elseif (!isset($this->dropTokens[$token[0]])) {
                     $value = $token[1];
                     $startAttributes['startLine'] = $token[2];

lib/PHPParser/Serializer/XML.php

@@ -46,6 +46,11 @@ class PHPParser_Serializer_XML implements PHPParser_Serializer
                 $this->writer->endElement();
             }
 
+            $this->writer->endElement();
+        } elseif ($node instanceof PHPParser_Comment) {
+            $this->writer->startElement('comment');
+            $this->writer->writeAttribute('isDocComment', $node instanceof PHPParser_Comment_Doc ? 'true' : 'false');
+            $this->writer->text($node->getText());
             $this->writer->endElement();
         } elseif (is_array($node)) {
             $this->writer->startElement('scalar:array');

test/PHPParser/Tests/LexerTest.php

@@ -49,9 +49,9 @@ class PHPParser_Tests_LexerTest extends PHPUnit_Framework_TestCase
 
     public function provideTestLex() {
         return array(
-            // tests conversion of closing PHP tag and drop of whitespace, comments and opening tags
+            // tests conversion of closing PHP tag and drop of whitespace and opening tags
             array(
-                '<?php tokens // ?>plaintext',
+                '<?php tokens ?>plaintext',
                 array(
                     array(
                         PHPParser_Parser::T_STRING, 'tokens',
@@ -81,17 +81,30 @@ class PHPParser_Tests_LexerTest extends PHPUnit_Framework_TestCase
                     ),
                     array(
                         ord('$'), '$',
-                        array('startLine' => 3, 'docComment' => '/** doc' . "\n" . 'comment */'), array('endLine' => 3)
+                        array(
+                            'startLine' => 3,
+                            'comments' => array(new PHPParser_Comment_Doc('/** doc' . "\n" . 'comment */'))
+                        ),
+                        array('endLine' => 3)
                     ),
                 )
             ),
-            // tests doccomment extraction
+            // tests comment extraction
             array(
-                '<?php /** docComment 1 *//** docComment 2 */ token',
+                '<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
                 array(
                     array(
                         PHPParser_Parser::T_STRING, 'token',
-                        array('startLine' => 1, 'docComment' => '/** docComment 2 */'), array('endLine' => 1)
+                        array(
+                            'startLine' => 2,
+                            'comments' => array(
+                                new PHPParser_Comment('/* comment */'),
+                                new PHPParser_Comment('// comment' . "\n"),
+                                new PHPParser_Comment_Doc('/** docComment 1 */'),
+                                new PHPParser_Comment_Doc('/** docComment 2 */'),
+                            ),
+                        ),
+                        array('endLine' => 2)
                     ),
                 )
             ),

test/PHPParser/Tests/Serializer/XMLTest.php

@@ -8,6 +8,7 @@ class PHPParser_Tests_Serializer_XMLTest extends PHPUnit_Framework_TestCase
     public function testSerialize() {
         $code = <<<CODE
 <?php
+// comment
 /** doc comment */
 function functionName(&\$a = 0, \$b = 1.0) {
     echo 'Foo';
@@ -18,14 +19,18 @@ CODE;
 <AST xmlns:node="http://nikic.github.com/PHPParser/XML/node" xmlns:subNode="http://nikic.github.com/PHPParser/XML/subNode" xmlns:attribute="http://nikic.github.com/PHPParser/XML/attribute" xmlns:scalar="http://nikic.github.com/PHPParser/XML/scalar">
  <scalar:array>
   <node:Stmt_Function>
-   <attribute:docComment>
+   <attribute:comments>
-    <scalar:string>/** doc comment */</scalar:string>
+    <scalar:array>
-   </attribute:docComment>
+     <comment isDocComment="false">// comment
+</comment>
+     <comment isDocComment="true">/** doc comment */</comment>
+    </scalar:array>
+   </attribute:comments>
    <attribute:startLine>
-    <scalar:int>3</scalar:int>
+    <scalar:int>4</scalar:int>
    </attribute:startLine>
    <attribute:endLine>
-    <scalar:int>5</scalar:int>
+    <scalar:int>6</scalar:int>
    </attribute:endLine>
    <subNode:byRef>
     <scalar:false/>
@@ -34,10 +39,10 @@ CODE;
     <scalar:array>
      <node:Param>
       <attribute:startLine>
-       <scalar:int>3</scalar:int>
+       <scalar:int>4</scalar:int>
       </attribute:startLine>
       <attribute:endLine>
-       <scalar:int>3</scalar:int>
+       <scalar:int>4</scalar:int>
       </attribute:endLine>
       <subNode:name>
        <scalar:string>a</scalar:string>
@@ -45,10 +50,10 @@ CODE;
       <subNode:default>
        <node:Scalar_LNumber>
         <attribute:startLine>
-         <scalar:int>3</scalar:int>
+         <scalar:int>4</scalar:int>
         </attribute:startLine>
         <attribute:endLine>
-         <scalar:int>3</scalar:int>
+         <scalar:int>4</scalar:int>
         </attribute:endLine>
         <subNode:value>
          <scalar:int>0</scalar:int>
@@ -64,10 +69,10 @@ CODE;
      </node:Param>
      <node:Param>
       <attribute:startLine>
-       <scalar:int>3</scalar:int>
+       <scalar:int>4</scalar:int>
       </attribute:startLine>
       <attribute:endLine>
-       <scalar:int>3</scalar:int>
+       <scalar:int>4</scalar:int>
       </attribute:endLine>
       <subNode:name>
        <scalar:string>b</scalar:string>
@@ -75,10 +80,10 @@ CODE;
       <subNode:default>
        <node:Scalar_DNumber>
         <attribute:startLine>
-         <scalar:int>3</scalar:int>
+         <scalar:int>4</scalar:int>
         </attribute:startLine>
         <attribute:endLine>
-         <scalar:int>3</scalar:int>
+         <scalar:int>4</scalar:int>
         </attribute:endLine>
         <subNode:value>
          <scalar:float>1</scalar:float>
@@ -98,19 +103,19 @@ CODE;
     <scalar:array>
      <node:Stmt_Echo>
       <attribute:startLine>
-       <scalar:int>4</scalar:int>
+       <scalar:int>5</scalar:int>
       </attribute:startLine>
       <attribute:endLine>
-       <scalar:int>4</scalar:int>
+       <scalar:int>5</scalar:int>
       </attribute:endLine>
       <subNode:exprs>
        <scalar:array>
         <node:Scalar_String>
          <attribute:startLine>
-          <scalar:int>4</scalar:int>
+          <scalar:int>5</scalar:int>
          </attribute:startLine>
          <attribute:endLine>
-          <scalar:int>4</scalar:int>
+          <scalar:int>5</scalar:int>
          </attribute:endLine>
          <subNode:value>
           <scalar:string>Foo</scalar:string>

test/PHPParser/Tests/Unserializer/XMLTest.php

@@ -39,8 +39,6 @@ XML;
 XML;
 
         $unserializer  = new PHPParser_Unserializer_XML;
-        var_dump($unserializer->unserialize($xml));
-
 
         $this->assertEquals(
             new PHPParser_Node_Scalar_ClassConst,