danog/psalm
1
0
Fork 0
mirror of https://github.com/danog/psalm.git synced 2025-01-22 05:41:20 +01:00
Code Issues Projects Releases Wiki Activity

Add support for forking at the scanning step

Browse Source
This commit is contained in:
Brown 2018-10-11 13:58:39 -04:00 committed by Matthew Brown
parent edc219facb
commit f9cca5e597
7 changed files with 364 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/Psalm/Checker/ProjectChecker.php
View File

@ -289,7 +289,7 @@ class ProjectChecker
$this->config->initializePlugins($this);
$this->codebase->scanFiles();
$this->codebase->scanFiles($this->threads);
} else {
if ($this->debug_output) {
echo count($diff_files) . ' changed files: ' . "\n";
@ -306,7 +306,7 @@ class ProjectChecker
$this->config->initializePlugins($this);
$this->codebase->scanFiles();
$this->codebase->scanFiles($this->threads);
}
}
@ -401,7 +401,7 @@ class ProjectChecker
$this->config->initializePlugins($this);
$this->codebase->scanFiles();
$this->codebase->scanFiles($this->threads);
$this->config->visitStubFiles($this->codebase, $this->debug_output);
@ -571,7 +571,7 @@ class ProjectChecker
$this->config->initializePlugins($this);
$this->codebase->scanFiles();
$this->codebase->scanFiles($this->threads);
$this->config->visitStubFiles($this->codebase, $this->debug_output);
@ -609,7 +609,7 @@ class ProjectChecker
$this->config->initializePlugins($this);
$this->codebase->scanFiles();
$this->codebase->scanFiles($this->threads);
$this->config->visitStubFiles($this->codebase, $this->debug_output);

4
src/Psalm/Codebase.php
View File

@ -292,9 +292,9 @@ class Codebase
*
* @return void
*/
public function scanFiles()
public function scanFiles(int $threads = 1)
{
$has_changes = $this->scanner->scanFiles($this->classlikes);
$has_changes = $this->scanner->scanFiles($this->classlikes, $threads);
if ($has_changes) {
$this->populator->populateCodebase($this);

73
src/Psalm/Codebase/ClassLikes.php
View File

@ -888,4 +888,77 @@ class ClassLikes
$this->scanner->removeClassLike($fq_class_name_lc);
}
/**
* @return array{
* 0: array<string, bool>,
* 1: array<string, bool>,
* 2: array<string, bool>,
* 3: array<string, bool>,
* 4: array<string, bool>,
* 5: array<string, bool>,
* 6: array<string, bool>,
* 7: array<string, \PhpParser\Node\Stmt\Trait_>,
* 8: array<string, \Psalm\Aliases>,
* 9: array<string, int>
* }
*/
public function getThreadData()
{
return [
$this->existing_classlikes_lc,
$this->existing_classes_lc,
$this->existing_traits_lc,
$this->existing_traits,
$this->existing_interfaces_lc,
$this->existing_interfaces,
$this->existing_classes,
$this->trait_nodes,
$this->trait_aliases,
$this->classlike_references
];
}
/**
* @param array{
* 0: array<string, bool>,
* 1: array<string, bool>,
* 2: array<string, bool>,
* 3: array<string, bool>,
* 4: array<string, bool>,
* 5: array<string, bool>,
* 6: array<string, bool>,
* 7: array<string, \PhpParser\Node\Stmt\Trait_>,
* 8: array<string, \Psalm\Aliases>,
* 9: array<string, int>
* } $thread_data
*
* @return void
*/
public function addThreadData(array $thread_data)
{
list (
$existing_classlikes_lc,
$existing_classes_lc,
$existing_traits_lc,
$existing_traits,
$existing_interfaces_lc,
$existing_interfaces,
$existing_classes,
$trait_nodes,
$trait_aliases,
$classlike_references
) = $thread_data;
$this->existing_classlikes_lc = array_merge($existing_classlikes_lc, $this->existing_classlikes_lc);
$this->existing_classes_lc = array_merge($existing_classes_lc, $this->existing_classes_lc);
$this->existing_traits_lc = array_merge($existing_traits_lc, $this->existing_traits_lc);
$this->existing_traits = array_merge($existing_traits, $this->existing_traits);
$this->existing_interfaces_lc = array_merge($existing_interfaces_lc, $this->existing_interfaces_lc);
$this->existing_interfaces = array_merge($existing_interfaces, $this->existing_interfaces);
$this->existing_classes = array_merge($existing_classes, $this->existing_classes);
$this->trait_nodes = array_merge($trait_nodes, $this->trait_nodes);
$this->trait_aliases = array_merge($trait_aliases, $this->trait_aliases);
$this->classlike_references = array_merge($classlike_references, $this->classlike_references);
}
}

247
src/Psalm/Codebase/Scanner.php
View File

@ -8,6 +8,57 @@ use Psalm\Provider\FileReferenceProvider;
use Psalm\Provider\FileStorageProvider;
use Psalm\Scanner\FileScanner;
/**
* @psalm-type IssueData = array{
* severity: string,
* line_from: int,
* line_to: int,
* type: string,
* message: string,
* file_name: string,
* file_path: string,
* snippet: string,
* from: int,
* to: int,
* snippet_from: int,
* snippet_to: int,
* column_from: int,
* column_to: int
* }
*
* @psalm-type PoolData = array{
* classlikes_data:array{
* 0:array<string, bool>,
* 1:array<string, bool>,
* 2:array<string, bool>,
* 3:array<string, bool>,
* 4:array<string, bool>,
* 5:array<string, bool>,
* 6:array<string, bool>,
* 7:array<string, \PhpParser\Node\Stmt\Trait_>,
* 8:array<string, \Psalm\Aliases>,
* 9:array<string, int>
* },
* scanner_data:array{
* 0:array<string, string>,
* 1:array<string, string>,
* 2:array<string, string>,
* 3:array<string, bool>,
* 4:array<string, bool>,
* 5:array<string, string>,
* 6:array<string, bool>,
* 7:array<string, bool>,
* 8:array<string, bool>
* },
* issues:array<int, IssueData>,
* changed_members:array<string, array<string, bool>>,
* unchanged_signature_members:array<string, array<string, bool>>,
* diff_map:array<string, array<int, array{0:int, 1:int, 2:int, 3:int}>>,
* classlike_storage:array<string, \Psalm\Storage\ClassLikeStorage>,
* file_storage:array<string, \Psalm\Storage\FileStorage>
* }
*/
/**
* @internal
*
@ -95,6 +146,11 @@ class Scanner
*/
private $file_reference_provider;
/**
* @var bool
*/
private $is_forked = false;
/**
* @param bool $debug_output
*/
@ -257,13 +313,13 @@ class Scanner
/**
* @return bool
*/
public function scanFiles(ClassLikes $classlikes)
public function scanFiles(ClassLikes $classlikes, int $pool_size = 1)
{
$has_changes = false;
while ($this->files_to_scan || $this->classes_to_scan) {
if ($this->files_to_scan) {
if ($this->scanFilePaths()) {
if ($this->scanFilePaths($pool_size)) {
$has_changes = true;
}
} else {
@ -274,7 +330,7 @@ class Scanner
return $has_changes;
}
private function scanFilePaths() : bool
private function scanFilePaths(int $pool_size) : bool
{
$filetype_scanners = $this->config->getFiletypeScanners();
$files_to_scan = array_filter(
@ -287,12 +343,108 @@ class Scanner
$this->files_to_scan = [];
foreach ($files_to_scan as $file_path) {
$this->scanFile(
$file_path,
$filetype_scanners,
isset($this->files_to_deep_scan[$file_path])
$files_to_deep_scan = $this->files_to_deep_scan;
$scanner_worker =
/**
* @param int $_
* @param string $file_path
*
* @return void
*/
function ($_, $file_path) use ($filetype_scanners, $files_to_deep_scan) {
$this->scanFile(
$file_path,
$filetype_scanners,
isset($files_to_deep_scan[$file_path])
);
};
if (!$this->is_forked && $pool_size > 1 && count($files_to_scan) > 512) {
$pool_size = ceil(min($pool_size, count($files_to_scan) / 256));
} else {
$pool_size = 1;
}
if ($pool_size > 1) {
$process_file_paths = [];
$i = 0;
foreach ($files_to_scan as $file_path) {
$process_file_paths[$i % $pool_size][] = $file_path;
++$i;
}
// Run scanning one file at a time, splitting the set of
// files up among a given number of child processes.
$pool = new \Psalm\Fork\Pool(
$process_file_paths,
/** @return void */
function () {
$project_checker = \Psalm\Checker\ProjectChecker::getInstance();
$statements_provider = $project_checker->codebase->statements_provider;
$project_checker->codebase->scanner->isForked();
$project_checker->codebase->file_storage_provider->deleteAll();
$project_checker->codebase->classlike_storage_provider->deleteAll();
$statements_provider->resetDiffs();
},
$scanner_worker,
/**
* @return PoolData
*/
function () {
$project_checker = \Psalm\Checker\ProjectChecker::getInstance();
$statements_provider = $project_checker->codebase->statements_provider;
return [
'classlikes_data' => $project_checker->codebase->classlikes->getThreadData(),
'scanner_data' => $project_checker->codebase->scanner->getThreadData(),
'issues' => \Psalm\IssueBuffer::getIssuesData(),
'changed_members' => $statements_provider->getChangedMembers(),
'unchanged_signature_members' => $statements_provider->getUnchangedSignatureMembers(),
'diff_map' => $statements_provider->getDiffMap(),
'classlike_storage' => $project_checker->classlike_storage_provider->getAll(),
'file_storage' => $project_checker->file_storage_provider->getAll(),
];
}
);
// Wait for all tasks to complete and collect the results.
/**
* @var array<int, PoolData>
*/
$forked_pool_data = $pool->wait();
foreach ($forked_pool_data as $pool_data) {
\Psalm\IssueBuffer::addIssues($pool_data['issues']);
$this->codebase->statements_provider->addChangedMembers(
$pool_data['changed_members']
);
$this->codebase->statements_provider->addUnchangedSignatureMembers(
$pool_data['unchanged_signature_members']
);
$this->codebase->statements_provider->addDiffMap(
$pool_data['diff_map']
);
$this->codebase->file_storage_provider->addMore($pool_data['file_storage']);
$this->codebase->classlike_storage_provider->addMore($pool_data['classlike_storage']);
$this->codebase->classlikes->addThreadData($pool_data['classlikes_data']);
$this->addThreadData($pool_data['scanner_data']);
}
} else {
$i = 0;
foreach ($files_to_scan as $file_path => $_) {
$scanner_worker($i, $file_path);
++$i;
}
}
return (bool) $files_to_scan;
@ -551,4 +703,83 @@ class Scanner
return true;
}
/**
* @return array{
* 0: array<string, string>,
* 1: array<string, string>,
* 2: array<string, string>,
* 3: array<string, bool>,
* 4: array<string, bool>,
* 5: array<string, string>,
* 6: array<string, bool>,
* 7: array<string, bool>,
* 8: array<string, bool>
* }
*/
public function getThreadData()
{
return [
$this->files_to_scan,
$this->files_to_deep_scan,
$this->classes_to_scan,
$this->classes_to_deep_scan,
$this->store_scan_failure,
$this->classlike_files,
$this->deep_scanned_classlike_files,
$this->scanned_files,
$this->reflected_classlikes_lc
];
}
/**
* @param array{
* 0: array<string, string>,
* 1: array<string, string>,
* 2: array<string, string>,
* 3: array<string, bool>,
* 4: array<string, bool>,
* 5: array<string, string>,
* 6: array<string, bool>,
* 7: array<string, bool>,
* 8: array<string, bool>
* } $thread_data
*
* @return void
*/
public function addThreadData(array $thread_data)
{
list(
$files_to_scan,
$files_to_deep_scan,
$classes_to_scan,
$classes_to_deep_scan,
$store_scan_failure,
$classlike_files,
$deep_scanned_classlike_files,
$scanned_files,
$reflected_classlikes_lc
) = $thread_data;
$this->files_to_scan = array_merge($files_to_scan, $this->files_to_scan);
$this->files_to_deep_scan = array_merge($files_to_deep_scan, $this->files_to_deep_scan);
$this->classes_to_scan = array_merge($classes_to_scan, $this->classes_to_scan);
$this->classes_to_deep_scan = array_merge($classes_to_deep_scan, $this->classes_to_deep_scan);
$this->store_scan_failure = array_merge($store_scan_failure, $this->store_scan_failure);
$this->classlike_files = array_merge($classlike_files, $this->classlike_files);
$this->deep_scanned_classlike_files = array_merge(
$deep_scanned_classlike_files,
$this->deep_scanned_classlike_files
);
$this->scanned_files = array_merge($scanned_files, $this->scanned_files);
$this->reflected_classlikes_lc = array_merge($reflected_classlikes_lc, $this->reflected_classlikes_lc);
}
/**
* @return void
*/
public function isForked()
{
$this->is_forked = true;
}
}

9
src/Psalm/Provider/ClassLikeStorageProvider.php
View File

@ -84,6 +84,15 @@ class ClassLikeStorageProvider
return self::$storage;
}
/**
* @param array<string, ClassLikeStorage> $more
* @return void
*/
public function addMore(array $more)
{
self::$storage = array_merge($more, self::$storage);
}
/**
* @param string $fq_classlike_name
*

9
src/Psalm/Provider/FileStorageProvider.php
View File

@ -86,6 +86,15 @@ class FileStorageProvider
return self::$storage;
}
/**
* @param array<string, FileStorage> $more
* @return void
*/
public function addMore(array $more)
{
self::$storage = array_merge($more, self::$storage);
}
/**
* @param string $file_path
*

27
src/Psalm/Provider/StatementsProvider.php
View File

@ -213,6 +213,15 @@ class StatementsProvider
return $this->changed_members;
}
/**
* @param array<string, array<string, bool>> $more_changed_members
* @return void
*/
public function addChangedMembers(array $more_changed_members)
{
$this->changed_members = array_merge($more_changed_members, $this->changed_members);
}
/**
* @return array<string, array<string, bool>>
*/
@ -221,6 +230,15 @@ class StatementsProvider
return $this->unchanged_signature_members;
}
/**
* @param array<string, array<string, bool>> $more_unchanged_members
* @return void
*/
public function addUnchangedSignatureMembers(array $more_unchanged_members)
{
$this->unchanged_signature_members = array_merge($more_unchanged_members, $this->unchanged_signature_members);
}
/**
* @param string $file_path
* @return void
@ -240,6 +258,15 @@ class StatementsProvider
return $this->diff_map;
}
/**
* @param array<string, array<int, array{0: int, 1: int, 2: int, 3: int}>> $diff_map
* @return void
*/
public function addDiffMap(array $diff_map)
{
$this->diff_map = array_merge($diff_map, $this->diff_map);
}
/**
* @return void
*/