Introduce issue baseline with --set-baseline and --with-baseline

config.xsd

@@ -43,6 +43,7 @@
         <xs:attribute name="addParamDefaultToDocblockType" type="xs:string" />
         <xs:attribute name="checkForThrowsDocblock" type="xs:string" />
         <xs:attribute name="forbidEcho" type="xs:string" />
+        <xs:attribute name="errorBaseline" type="xs:string" />
     </xs:complexType>
 
     <xs:complexType name="ProjectFilesType">

psalm.xml.dist

@@ -24,6 +24,7 @@
             <file name="src/Psalm/Traverser/CustomTraverser.php" />
             <directory name="tests/performance/a.test" />
             <directory name="tests/performance/b.test" />
+            <file name="tests/ErrorBaselineTest.php" />
         </ignoreFiles>
     </projectFiles>
 

src/Psalm/Config.php

@@ -301,6 +301,9 @@ class Config
      */
     public $modified_time = 0;
 
+    /** @var string|null */
+    public $error_baseline = null;
+
     protected function __construct()
     {
         self::$instance = $this;
@@ -573,6 +576,11 @@ class Config
             $config->forbid_echo = $attribute_text === 'true' || $attribute_text === '1';
         }
 
+        if (isset($config_xml['errorBaseline'])) {
+            $attribute_text = (string) $config_xml['errorBaseline'];
+            $config->error_baseline = $attribute_text;
+        }
+
         if (isset($config_xml->projectFiles)) {
             $config->project_files = ProjectFileFilter::loadFromXMLElement($config_xml->projectFiles, $base_dir, true);
         }

src/Psalm/ErrorBaseline.php

@@ -0,0 +1,183 @@
+<?php
+namespace Psalm;
+
+use Psalm\Provider\FileProvider;
+
+class ErrorBaseline
+{
+    /**
+     * @param FileProvider $fileProvider
+     * @param string $baselineFile
+     * @param array<array{file_name: string, type: string, severity: string}> $issues
+     *
+     * @return void
+     */
+    public static function create(FileProvider $fileProvider, string $baselineFile, array $issues)
+    {
+        $groupedIssues = self::countIssueTypesByFile($issues);
+
+        self::writeToFile($fileProvider, $baselineFile, $groupedIssues);
+    }
+
+    /**
+     * @param FileProvider $fileProvider
+     * @param string $baselineFile
+     * @return array<string,array<string,int>>
+     * @throws Exception\ConfigException
+     */
+    public static function read(FileProvider $fileProvider, string $baselineFile): array
+    {
+        if (!$fileProvider->fileExists($baselineFile)) {
+            throw new Exception\ConfigException("{$baselineFile} does not exist or is not readable\n");
+        }
+
+        $xmlSource = $fileProvider->getContents($baselineFile);
+
+        $baselineDoc = new \DOMDocument();
+        $baselineDoc->loadXML($xmlSource, LIBXML_NOBLANKS);
+
+        /** @var \DOMNodeList $filesElement */
+        $filesElement = $baselineDoc->getElementsByTagName('files');
+
+        if ($filesElement->length === 0) {
+            throw new Exception\ConfigException('Baseline file does not contain <files>');
+        }
+
+        $files = [];
+
+        /** @var \DOMElement $filesElement */
+        $filesElement = $filesElement[0];
+
+        /** @var \DOMElement $file */
+        foreach ($filesElement->getElementsByTagName('file') as $file) {
+            $fileName = $file->getAttribute('src');
+
+            $files[$fileName] = [];
+
+            /** @var \DOMElement $issue */
+            foreach ($file->childNodes as $issue) {
+                $issueType = $issue->tagName;
+
+                $files[$fileName][$issueType] = (int)$issue->getAttribute('occurrences');
+            }
+        }
+
+        return $files;
+    }
+
+    /**
+     * @param FileProvider $fileProvider
+     * @param string $baselineFile
+     * @param array<array{file_name: string, type: string, severity: string}> $issues
+     * @return array<string,array<string,int>>
+     * @throws Exception\ConfigException
+     */
+    public static function update(FileProvider $fileProvider, string $baselineFile, array $issues)
+    {
+        $existingIssues = self::read($fileProvider, $baselineFile);
+        $newIssues = self::countIssueTypesByFile($issues);
+
+        foreach ($existingIssues as $file => &$existingIssuesCount) {
+            if (!isset($newIssues[$file])) {
+                unset($existingIssues[$file]);
+
+                continue;
+            }
+
+            foreach ($existingIssuesCount as $issueType => $count) {
+                if (!isset($newIssues[$file][$issueType])) {
+                    unset($existingIssuesCount[$issueType]);
+
+                    continue;
+                }
+
+                $existingIssuesCount[$issueType] = min($count, $newIssues[$file][$issueType]);
+            }
+        }
+
+        $groupedIssues = array_filter($existingIssues);
+
+        self::writeToFile($fileProvider, $baselineFile, $groupedIssues);
+
+        return $groupedIssues;
+    }
+
+    /**
+     * @param array<array{file_name: string, type: string, severity: string}> $issues
+     * @return array<string,array<string,int>>
+     */
+    private static function countIssueTypesByFile(array $issues): array
+    {
+        $groupedIssues = array_reduce(
+            $issues,
+            /**
+             * @param array<string,array<string,int>> $carry
+             * @param array{type: string, file_name: string, severity: string} $issue
+             * @return array<string,array<string,int>>
+             */
+            function (array $carry, array $issue): array {
+                if ($issue['severity'] !== Config::REPORT_ERROR) {
+                    return $carry;
+                }
+
+                $fileName = $issue['file_name'];
+                $issueType = $issue['type'];
+
+                if (!isset($carry[$fileName])) {
+                    $carry[$fileName] = [];
+                }
+
+                if (!isset($carry[$fileName][$issueType])) {
+                    $carry[$fileName][$issueType] = 0;
+                }
+
+                $carry[$fileName][$issueType]++;
+
+                return $carry;
+            },
+            []
+        );
+
+        // Sort files first
+        ksort($groupedIssues);
+
+        foreach ($groupedIssues as &$issues) {
+            ksort($issues);
+        }
+
+        return $groupedIssues;
+    }
+
+    /**
+     * @param FileProvider $fileProvider
+     * @param string $baselineFile
+     * @param array<string,array<string,int>> $groupedIssues
+     * @return void
+     */
+    private static function writeToFile(
+        FileProvider $fileProvider,
+        string $baselineFile,
+        array $groupedIssues
+    ) {
+        $baselineDoc = new \DOMDocument('1.0', 'UTF-8');
+        $filesNode = $baselineDoc->createElement('files');
+
+        foreach ($groupedIssues as $file => $issueTypes) {
+            $fileNode = $baselineDoc->createElement('file');
+            $fileNode->setAttribute('src', $file);
+
+            foreach ($issueTypes as $issueType => $count) {
+                $issueNode = $baselineDoc->createElement($issueType);
+                $issueNode->setAttribute('occurrences', (string)$count);
+                $fileNode->appendChild($issueNode);
+            }
+
+            $filesNode->appendChild($fileNode);
+        }
+
+        $baselineDoc->appendChild($filesNode);
+        $baselineDoc->formatOutput = true;
+
+        $fileProvider->setContents($baselineFile, $baselineDoc->saveXML());
+    }
+}

src/Psalm/IssueBuffer.php

@@ -304,14 +304,16 @@ class IssueBuffer
      * @param  bool                             $is_full
      * @param  float                            $start_time
      * @param  bool                             $add_stats
+     * @param  array<string,array<string,int>>  $issue_baseline
      *
      * @return void
      */
     public static function finish(
         ProjectChecker $project_checker,
-        $is_full,
+        bool $is_full,
-        $start_time,
+        float $start_time,
-        $add_stats = false
+        bool $add_stats = false,
+        array $issue_baseline = []
     ) {
         if ($project_checker->output_format === ProjectChecker::TYPE_CONSOLE) {
             echo "\n";
@@ -341,6 +343,21 @@ class IssueBuffer
                 }
             );
 
+            if (!empty($issue_baseline)) {
+                // Set severity for issues in baseline to INFO
+                foreach (self::$issues_data as $key => $issue_data) {
+                    $file = $issue_data['file_name'];
+                    $type = $issue_data['type'];
+
+                    if (isset($issue_baseline[$file][$type]) && $issue_baseline[$file][$type] > 0) {
+                        $issue_data['severity'] = Config::REPORT_INFO;
+                        $issue_baseline[$file][$type] = (int)$issue_baseline[$file][$type] - 1;
+                    }
+
+                    self::$issues_data[$key] = $issue_data;
+                }
+            }
+
             foreach (self::$issues_data as $issue_data) {
                 if ($issue_data['severity'] === Config::REPORT_ERROR) {
                     ++$error_count;

src/psalm.php

@@ -1,6 +1,7 @@
 <?php
 require_once('command_functions.php');
 
+use Psalm\ErrorBaseline;
 use Psalm\Checker\ProjectChecker;
 use Psalm\Config;
 use Psalm\IssueBuffer;
@@ -25,10 +26,12 @@ $valid_long_options = [
     'debug',
     'debug-by-line',
     'diff',
+    'diff-methods',
     'disable-extension:',
     'find-dead-code',
     'find-references-to:',
     'help',
+    'ignore-baseline',
     'init',
     'monochrome',
     'no-cache',
@@ -36,13 +39,14 @@ $valid_long_options = [
     'plugin:',
     'report:',
     'root:',
+    'set-baseline:',
     'show-info:',
     'show-snippet:',
     'stats',
     'threads:',
+    'update-baseline',
     'use-ini-defaults',
     'version',
-    'diff-methods',
 ];
 
 $args = array_slice($argv, 1);
@@ -190,6 +194,15 @@ Options:
     --disable-extension=[extension]
         Used to disable certain extensions while Psalm is running.
         
+    --set-baseline=PATH
+        Save all current error level issues to a file, to mark them as info in subsequent runs
+
+    --ignore-baseline=PATH
+        Ignore the error baseline
+        
+    --update-baseline
+        Update the baseline by removing fixed issues. This will not add new issues to the baseline
+
 HELP;
 
     exit;
@@ -252,6 +265,12 @@ $ini_handler->check();
 
 setlocale(LC_CTYPE, 'C');
 
+if (isset($options['set-baseline'])) {
+    if (is_array($options['set-baseline'])) {
+        die('Only one baseline file can be created at a time' . PHP_EOL);
+    }
+}
+
 if (isset($options['i'])) {
     if (file_exists($current_dir . 'psalm.xml')) {
         die('A config file already exists in the current directory' . PHP_EOL);
@@ -477,4 +496,59 @@ if ($find_references_to) {
     }
 }
 
-IssueBuffer::finish($project_checker, !$paths_to_check, $start_time, isset($options['stats']));
+if (isset($options['set-baseline']) && is_string($options['set-baseline'])) {
+    echo 'Writing error baseline to file...', PHP_EOL;
+
+    ErrorBaseline::create(
+        new \Psalm\Provider\FileProvider,
+        $options['set-baseline'],
+        IssueBuffer::getIssuesData()
+    );
+
+    echo "Baseline saved to {$options['set-baseline']}.";
+
+    if (Config::getInstance()->error_baseline !== $options['set-baseline']) {
+        echo " Don't forget to set errorBaseline=\"{$options['set-baseline']}\" in your config.";
+    }
+
+    echo PHP_EOL;
+}
+
+$issue_baseline = [];
+
+if (isset($options['update-baseline'])) {
+    $baselineFile = Config::getInstance()->error_baseline;
+
+    if (empty($baselineFile)) {
+        die('Cannot update baseline, because no baseline file is configured.' . PHP_EOL);
+    }
+
+    try {
+        $issue_baseline = ErrorBaseline::update(
+            new \Psalm\Provider\FileProvider,
+            $baselineFile,
+            IssueBuffer::getIssuesData()
+        );
+    } catch (\Psalm\Exception\ConfigException $exception) {
+        die('Could not update baseline file: ' . $exception->getMessage());
+    }
+}
+
+if (!empty(Config::getInstance()->error_baseline) && !isset($options['ignore-baseline'])) {
+    try {
+        $issue_baseline = ErrorBaseline::read(
+            new \Psalm\Provider\FileProvider,
+            (string)Config::getInstance()->error_baseline
+        );
+    } catch (\Psalm\Exception\ConfigException $exception) {
+        die('Error while reading baseline: ' . $exception->getMessage());
+    }
+}
+
+IssueBuffer::finish(
+    $project_checker,
+    !$paths_to_check,
+    $start_time,
+    isset($options['stats']),
+    $issue_baseline
+);

tests/ErrorBaselineTest.php

@@ -0,0 +1,257 @@
+<?php
+namespace Psalm\Tests;
+
+use Prophecy\Argument;
+use Prophecy\Prophecy\ObjectProphecy;
+use Psalm\ErrorBaseline;
+use Psalm\Exception\ConfigException;
+use Psalm\Provider\FileProvider;
+
+class ErrorBaselineTest extends TestCase
+{
+    /** @var ObjectProphecy */
+    private $fileProvider;
+
+    /**
+     * @return void
+     */
+    public function setUp()
+    {
+        $this->fileProvider = $this->prophesize(FileProvider::class);
+    }
+
+    /**
+     * @return void
+     */
+    public function testLoadShouldParseXmlBaselineToPhpArray()
+    {
+        $baselineFilePath = 'baseline.xml';
+
+        $this->fileProvider->fileExists($baselineFilePath)->willReturn(true);
+        $this->fileProvider->getContents($baselineFilePath)->willReturn(
+            '<?xml version="1.0" encoding="UTF-8"?>
+            <files>
+              <file src="sample/sample-file.php">
+                <MixedAssignment occurrences="2"/>
+                <InvalidReturnStatement occurrences="1"/>
+              </file>
+              <file src="sample/sample-file2.php">
+                <PossiblyUnusedMethod occurrences="2"/>
+              </file>
+            </files>'
+        );
+
+        $expectedParsedBaseline = [
+            'sample/sample-file.php' => [
+                'MixedAssignment' => 2,
+                'InvalidReturnStatement' => 1,
+            ],
+            'sample/sample-file2.php' => [
+                'PossiblyUnusedMethod' => 2,
+            ],
+        ];
+
+        $this->assertEquals(
+            $expectedParsedBaseline,
+            ErrorBaseline::read($this->fileProvider->reveal(), $baselineFilePath)
+        );
+    }
+
+    /**
+     * @return void
+     */
+    public function testLoadShouldThrowExceptionWhenFilesAreNotDefinedInBaselineFile()
+    {
+        $this->expectException(ConfigException::class);
+
+        $baselineFile = 'baseline.xml';
+
+        $this->fileProvider->fileExists($baselineFile)->willReturn(true);
+        $this->fileProvider->getContents($baselineFile)->willReturn(
+            '<?xml version="1.0" encoding="UTF-8"?>
+             <other>
+             </other>
+            '
+        );
+
+        ErrorBaseline::read($this->fileProvider->reveal(), $baselineFile);
+    }
+
+    /**
+     * @return void
+     */
+    public function testLoadShouldThrowExceptionWhenBaselineFileDoesNotExist()
+    {
+        $this->expectException(ConfigException::class);
+
+        $baselineFile = 'baseline.xml';
+
+        $this->fileProvider->fileExists($baselineFile)->willReturn(false);
+
+        ErrorBaseline::read($this->fileProvider->reveal(), $baselineFile);
+    }
+
+    /**
+     * @return void
+     */
+    public function testCreateShouldAggregateIssuesPerFile()
+    {
+        $baselineFile = 'baseline.xml';
+
+        $documentContent = null;
+
+        $this->fileProvider->setContents(
+            $baselineFile,
+            Argument::that(function (string $document) use (&$documentContent): bool {
+                $documentContent = $document;
+
+                return true;
+            })
+        )->willReturn(null);
+
+        ErrorBaseline::create($this->fileProvider->reveal(), $baselineFile, [
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedAssignment',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedAssignment',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedAssignment',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedOperand',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'AssignmentToVoid',
+                'severity' => 'info',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'CircularReference',
+                'severity' => 'suppress',
+            ],
+            [
+                'file_name' => 'sample/sample-file2.php',
+                'type' => 'MixedAssignment',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file2.php',
+                'type' => 'MixedAssignment',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file2.php',
+                'type' => 'TypeCoercion',
+                'severity' => 'error',
+            ],
+        ]);
+
+        $baselineDocument = new \DOMDocument();
+        $baselineDocument->loadXML($documentContent, LIBXML_NOBLANKS);
+
+        /** @var \DOMElement[] $files */
+        $files = $baselineDocument->getElementsByTagName('files')[0]->childNodes;
+
+        $file1 = $files[0];
+        $file2 = $files[1];
+        $this->assertEquals('sample/sample-file.php', $file1->getAttribute('src'));
+        $this->assertEquals('sample/sample-file2.php', $file2->getAttribute('src'));
+
+        /** @var \DOMElement[] $file1Issues */
+        $file1Issues = $file1->childNodes;
+        /** @var \DOMElement[] $file2Issues */
+        $file2Issues = $file2->childNodes;
+
+        $this->assertEquals('MixedAssignment', $file1Issues[0]->tagName);
+        $this->assertEquals(3, $file1Issues[0]->getAttribute('occurrences'));
+        $this->assertEquals('MixedOperand', $file1Issues[1]->tagName);
+        $this->assertEquals(1, $file1Issues[1]->getAttribute('occurrences'));
+
+        $this->assertEquals('MixedAssignment', $file2Issues[0]->tagName);
+        $this->assertEquals(2, $file2Issues[0]->getAttribute('occurrences'));
+        $this->assertEquals('TypeCoercion', $file2Issues[1]->tagName);
+        $this->assertEquals(1, $file2Issues[1]->getAttribute('occurrences'));
+    }
+
+    /**
+     * @return void
+     */
+    public function testUpdateShouldRemoveExistingIssuesWithoutAddingNewOnes()
+    {
+        $baselineFile = 'baseline.xml';
+
+        $this->fileProvider->fileExists($baselineFile)->willReturn(true);
+        $this->fileProvider->getContents($baselineFile)->willReturn(
+            '<?xml version="1.0" encoding="UTF-8"?>
+            <files>
+              <file src="sample/sample-file.php">
+                <MixedAssignment occurrences="3"/>
+                <MixedOperand occurrences="1"/>
+              </file>
+              <file src="sample/sample-file2.php">
+                <MixedAssignment occurrences="2"/>
+                <TypeCoercion occurrences="1"/>
+              </file>
+              <file src="sample/sample-file3.php">
+                <MixedAssignment occurrences="1"/>
+              </file>
+            </files>'
+        );
+        $this->fileProvider->setContents(Argument::cetera())->willReturn(null);
+
+        $newIssues = [
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedAssignment',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedAssignment',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedOperand',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file.php',
+                'type' => 'MixedOperand',
+                'severity' => 'error',
+            ],
+            [
+                'file_name' => 'sample/sample-file2.php',
+                'type' => 'TypeCoercion',
+                'severity' => 'error',
+            ],
+        ];
+
+        $remainingBaseline = ErrorBaseline::update(
+            $this->fileProvider->reveal(),
+            $baselineFile,
+            $newIssues
+        );
+
+        $this->assertEquals([
+            'sample/sample-file.php' => [
+                'MixedAssignment' => 2,
+                'MixedOperand' => 1,
+            ],
+            'sample/sample-file2.php' => [
+                'TypeCoercion' => 1,
+            ],
+        ], $remainingBaseline);
+    }
+}