Split unit test over multiple jobs in GitHub Actions, lowering the total run time (#4985)

* Split unit test over multiple jobs in GitHub Actions, lowering the total run time

* rename FileManipulationTest to FileManipulationTestCase so it does not run as a standalone Test

.github/workflows/ci.yml

@@ -2,8 +2,42 @@ name: Run unit tests
 
 on: [push, pull_request]
 jobs:
-  build:
+  chunk-matrix:
+    name: Generate Chunk Matrix
+
     runs-on: ubuntu-latest
+    env:
+      CHUNK_COUNT: 5
+
+    outputs:
+      count: ${{ steps.chunk-matrix.outputs.count }}
+      chunks: ${{ steps.chunk-matrix.outputs.chunks }}
+
+    steps:
+      - id: chunk-matrix
+        name: Generates the Chunk Matrix
+        run: |
+          echo "::set-output name=count::$(php -r 'echo json_encode([ ${{ env.CHUNK_COUNT }} ]);')"
+          echo "::set-output name=chunks::$(php -r 'echo json_encode(range(1, ${{ env.CHUNK_COUNT }} ));')"
+
+  tests:
+    name: "Unit Tests - ${{ matrix.chunk }}"
+
+    runs-on: ubuntu-latest
+    needs:
+      - chunk-matrix
+
+    strategy:
+      fail-fast: false
+      matrix:
+        count: ${{ fromJson(needs.chunk-matrix.outputs.count) }}
+        chunk: ${{ fromJson(needs.chunk-matrix.outputs.chunks) }}
+
+    env:
+      CHUNK_COUNT: "${{ matrix.count }}"
+      CHUNK_NUMBER: "${{ matrix.chunk }}"
+      PARALLEL_PROCESSES: 5
+
     steps:
       - name: Set up PHP
         uses: shivammathur/setup-php@v2
@@ -35,4 +69,4 @@ jobs:
           COMPOSER_ROOT_VERSION: dev-master
 
       - name: Run unit tests
-        run: php vendor/bin/paratest --runner=WrapperRunner
+        run: bin/tests-github-actions.sh

bin/tests-github-actions.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -eu
+
+function get_seeded_random() {
+  openssl enc -aes-256-ctr -pass pass:"vimeo/psalm" -nosalt </dev/zero 2>/dev/null
+}
+
+function run {
+  local -r chunk_count="$1"
+  local -r chunk_number="$2"
+  local -r parallel_processes="$3"
+
+  local -r phpunit_cmd='
+echo "::group::{}";
+vendor/bin/phpunit --log-junit build/phpunit/logs/{_}.xml --colors=always {};
+exit_code=$?;
+echo ::endgroup::;
+if [[ "$exit_code" -ne 0 ]]; then
+    echo "::error::{}";
+fi;
+exit "$exit_code"'
+
+  mkdir -p build/parallel/ build/phpunit/logs/
+
+  find tests -name '*Test.php' | shuf --random-source=<(get_seeded_random) > build/tests_all
+  split --number="l/$chunk_number/$chunk_count" build/tests_all > build/tests_split
+  parallel --group -j"$parallel_processes" --rpl {_}\ s/\\//_/g --joblog build/parallel/jobs.log "$phpunit_cmd" < build/tests_split
+}
+
+if [ -z "${CHUNK_COUNT:-}" ]; then echo "Did not find env var CHUNK_COUNT."; exit 1; fi
+if [ -z "${CHUNK_NUMBER:-}" ]; then echo "Did not find env var CHUNK_NUMBER."; exit 1; fi
+if [ -z "${PARALLEL_PROCESSES:-}" ]; then echo "Did not find env var PARALLEL_PROCESSES."; exit 1; fi
+
+run "$CHUNK_COUNT" "$CHUNK_NUMBER" "$PARALLEL_PROCESSES"

tests/FileManipulation/FileManipulationTestCase.php

@@ -7,7 +7,7 @@ use Psalm\Tests\Internal\Provider;
 use Psalm\Tests\TestConfig;
 use function strpos;
 
-abstract class FileManipulationTest extends \Psalm\Tests\TestCase
+abstract class FileManipulationTestCase extends \Psalm\Tests\TestCase
 {
     /** @var \Psalm\Internal\Analyzer\ProjectAnalyzer */
     protected $project_analyzer;

tests/FileManipulation/ImmutableAnnotationAdditionTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class ImmutableAnnotationAdditionTest extends FileManipulationTest
+class ImmutableAnnotationAdditionTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>

tests/FileManipulation/MissingPropertyTypeTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class MissingPropertyTypeTest extends FileManipulationTest
+class MissingPropertyTypeTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool,5?:bool}>

tests/FileManipulation/MissingReturnTypeTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class MissingReturnTypeTest extends FileManipulationTest
+class MissingReturnTypeTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool,5?:bool}>

tests/FileManipulation/ParamNameMismatchTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class ParamNameMismatchTest extends FileManipulationTest
+class ParamNameMismatchTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>

tests/FileManipulation/ParamTypeManipulationTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class ParamTypeManipulationTest extends FileManipulationTest
+class ParamTypeManipulationTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>

tests/FileManipulation/PureAnnotationAdditionTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class PureAnnotationAdditionTest extends FileManipulationTest
+class PureAnnotationAdditionTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>

tests/FileManipulation/ReturnTypeManipulationTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class ReturnTypeManipulationTest extends FileManipulationTest
+class ReturnTypeManipulationTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool,5?:bool}>

tests/FileManipulation/UndefinedVariableManipulationTest.php

@@ -3,7 +3,7 @@ namespace Psalm\Tests\FileManipulation;
 
 use const PHP_VERSION;
 
-class UndefinedVariableManipulationTest extends FileManipulationTest
+class UndefinedVariableManipulationTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>

tests/FileManipulation/UnnecessaryVarAnnotationManipulationTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class UnnecessaryVarAnnotationManipulationTest extends FileManipulationTest
+class UnnecessaryVarAnnotationManipulationTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>

tests/FileManipulation/UnusedCodeManipulationTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class UnusedCodeManipulationTest extends FileManipulationTest
+class UnusedCodeManipulationTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>

tests/FileManipulation/UnusedVariableManipulationTest.php

@@ -1,7 +1,7 @@
 <?php
 namespace Psalm\Tests\FileManipulation;
 
-class UnusedVariableManipulationTest extends FileManipulationTest
+class UnusedVariableManipulationTest extends FileManipulationTestCase
 {
     /**
      * @return array<string,array{string,string,string,string[],bool}>