From 60345201150b304c9163227b4bb70a60ba1d3817 Mon Sep 17 00:00:00 2001 From: "ondrej@mirtes.cz" Date: Fri, 6 Nov 2026 17:26:02 +0000 Subject: [PATCH] Measure benchmarks by counted instructions with CodSpeed instead of phpbench Counts the executed instructions of only the benchmark file's analysis under CodSpeed's CPU simulation, so the numbers no longer depend on the runner. Compares them with tests/bench/expected.neon in a sharded Benchmarks workflow and removes phpbench. Amp-Thread-ID: https://ampcode.com/threads/T-01a1ad6c-dec2-72ff-bbe8-5b3b62746c94 --- .agents/setup | 2 +- .github/workflows/bench.yml | 93 -- .github/workflows/benchmarks.yml | 140 +++ .github/workflows/tests.yml | 2 +- .gitignore | 2 - CLAUDE.md | 3 +- Makefile | 11 +- build/downgrade.php | 1 + build/ignore-by-php-version.neon.php | 1 + build/memory-reset-peak-usage.neon | 8 + build/phpstan.neon | 3 +- phpbench.json | 22 - .../Benchmark/BenchmarkComparatorTest.php | 79 ++ .../Benchmark/BenchmarkProfileParserTest.php | 81 ++ .../PHPStan/Benchmark/BenchmarkReportTest.php | 52 + .../PHPStan/Benchmark/BenchmarkShardsTest.php | 36 + tests/bench/BenchCase.php | 39 - tests/bench/BenchmarkComparator.php | 120 +++ tests/bench/BenchmarkProfileParser.php | 70 ++ tests/bench/BenchmarkReport.php | 390 +++++++ tests/bench/BenchmarkShards.php | 52 + tests/bench/RegressionBench.php | 51 - tests/bench/Warmup.php | 80 ++ tests/bench/compare.php | 78 ++ tests/bench/expected.neon | 468 +++++++++ tests/bench/measure.php | 138 +++ tests/bench/phpstan.neon | 9 + tests/bench/run.php | 153 +++ tests/bench/shard.php | 45 + tests/bench/storage/baseline.xml | 965 ------------------ tests/composer.json | 3 +- tests/composer.lock | 727 +------------ 32 files changed, 2011 insertions(+), 1913 deletions(-) delete mode 100644 .github/workflows/bench.yml create mode 100644 .github/workflows/benchmarks.yml create mode 100644 build/memory-reset-peak-usage.neon delete mode 100644 phpbench.json create mode 100644 tests/PHPStan/Benchmark/BenchmarkComparatorTest.php create mode 100644 tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php create mode 100644 tests/PHPStan/Benchmark/BenchmarkReportTest.php create mode 100644 tests/PHPStan/Benchmark/BenchmarkShardsTest.php delete mode 100644 tests/bench/BenchCase.php create mode 100644 tests/bench/BenchmarkComparator.php create mode 100644 tests/bench/BenchmarkProfileParser.php create mode 100644 tests/bench/BenchmarkReport.php create mode 100644 tests/bench/BenchmarkShards.php delete mode 100644 tests/bench/RegressionBench.php create mode 100644 tests/bench/Warmup.php create mode 100644 tests/bench/compare.php create mode 100644 tests/bench/expected.neon create mode 100644 tests/bench/measure.php create mode 100644 tests/bench/phpstan.neon create mode 100644 tests/bench/run.php create mode 100644 tests/bench/shard.php delete mode 100644 tests/bench/storage/baseline.xml diff --git a/.agents/setup b/.agents/setup index 4c132a9391d..dee48cc1d96 100755 --- a/.agents/setup +++ b/.agents/setup @@ -75,7 +75,7 @@ log "Using $(composer --version 2>/dev/null | head -1)" # the same lockfile. Patches are applied by cweagans/composer-patches on install. log "composer install (root)" composer install --no-interaction --no-progress -log "composer install (tests: paratest, phpbench, phpunit)" +log "composer install (tests: paratest, phpunit)" composer install --working-dir tests --no-interaction --no-progress log "setup complete" diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml deleted file mode 100644 index 0a0e7b77481..00000000000 --- a/.github/workflows/bench.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: "Benchmark" - -on: - pull_request: - paths: - - 'src/**' - - '.github/workflows/bench.yml' - - 'tests/bench/**' - push: - branches: - - "2.3.x" - paths: - - 'src/**' - - '.github/workflows/bench.yml' - - 'tests/bench/**' - -concurrency: - group: bench-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - baseline: - name: "Baseline" - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 - with: - egress-policy: audit - - - name: "Checkout base branch" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: "Install PHP" - uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0 - with: - coverage: "none" - php-version: "8.5" - tools: pecl - extensions: ds,mbstring - ini-file: development - ini-values: memory_limit=-1 - - - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0 - - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0 - with: - working-directory: "tests/" - - - name: "Run phpbench baseline" - run: "tests/vendor/bin/phpbench run --dump-file=tests/bench/storage/baseline.xml --ansi" - - - name: "Upload baseline artifact" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: phpbench-baseline - path: tests/bench/storage/baseline.xml - - test: - name: "Test" - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 - with: - egress-policy: audit - - - name: "Checkout" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: "Install PHP" - uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0 - with: - coverage: "none" - php-version: "8.5" - tools: pecl - extensions: ds,mbstring - ini-file: development - ini-values: memory_limit=-1 - - - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0 - - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0 - with: - working-directory: "tests/" - - - name: "Run phpbench test" - run: "tests/vendor/bin/phpbench run --file=tests/bench/storage/baseline.xml --report=my-report --ansi" diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000000..fc5a5cd290a --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,140 @@ +name: "Benchmarks" + +on: + pull_request: + paths: + - 'src/**' + - 'conf/**' + - 'stubs/**' + - 'composer.lock' + - 'tests/bench/**' + - '.github/workflows/benchmarks.yml' + push: + branches: + - "2.3.x" + paths: + - 'src/**' + - 'conf/**' + - 'stubs/**' + - 'composer.lock' + - 'tests/bench/**' + - '.github/workflows/benchmarks.yml' + +concurrency: + group: benchmarks-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CODSPEED_VERSION: "5.6.3" + # tests/bench/measure.php calls it over FFI to measure only the analysis of the benchmark + INSTRUMENT_HOOKS_COMMIT: "8fd5b878499f819561b33219cdafaa6f8c28b4e4" + +jobs: + run: + name: "Benchmarks shard ${{ matrix.shard }}" + # Pinned because the instruction counts depend on the PHP build and the system libraries + runs-on: ubuntu-24.04 + # The slowest shard takes about 20 minutes, and run.php stops a benchmark after 25 + timeout-minutes: 40 + + strategy: + fail-fast: false + matrix: + # Each shard is one runner, see tests/bench/shard.php + shard: [1, 2, 3, 4] + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 + with: + egress-policy: audit + + - name: "Checkout" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: "Install PHP" + uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0 + with: + coverage: "none" + php-version: "8.5" + extensions: ds,ffi,mbstring + ini-file: production + ini-values: memory_limit=-1 + + - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0 + + - name: "Install CodSpeed" + run: | + curl -fsSL "https://github.com/CodSpeedHQ/codspeed/releases/download/v${CODSPEED_VERSION}/codspeed-runner-installer.sh" | bash + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + "$HOME/.cargo/bin/codspeed" setup --mode=simulation + + - name: "Build CodSpeed's instrument-hooks" + run: | + git init -q "$RUNNER_TEMP/instrument-hooks" + cd "$RUNNER_TEMP/instrument-hooks" + git fetch -q --depth=1 https://github.com/CodSpeedHQ/instrument-hooks.git "$INSTRUMENT_HOOKS_COMMIT" + git checkout -q FETCH_HEAD + gcc -shared -fPIC -O2 -Iincludes -w dist/core.c -o libinstrument_hooks.so + + - name: "Run benchmarks" + env: + INSTRUMENT_HOOKS_LIBRARY: ${{ runner.temp }}/instrument-hooks/libinstrument_hooks.so + SHARD: ${{ matrix.shard }} + SHARDS: ${{ strategy.job-total }} + # Valgrind runs each benchmark on one CPU, so the shard runs as many at once as the runner has + run: | + mkdir -p tmp/bench-results + php tests/bench/shard.php "$SHARDS" "$SHARD" | xargs -P "$(nproc)" -I {} php tests/bench/run.php "tests/bench/data/{}" "tmp/bench-results/{}.neon" + + - name: "Upload results" + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: benchmark-results-${{ matrix.shard }} + path: tmp/bench-results/*.neon + if-no-files-found: ignore + + compare: + name: "Compare with expected.neon" + needs: run + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 + with: + egress-policy: audit + + - name: "Checkout" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: "Install PHP" + uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0 + with: + coverage: "none" + php-version: "8.5" + + - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0 + + - name: "Download results" + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: benchmark-results-* + merge-multiple: true + path: tmp/bench-results + + - name: "Compare" + run: "php tests/bench/compare.php tmp/bench-results tmp/expected.neon" + + - name: "Upload measured metrics" + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: benchmark-measured + path: tmp/expected.neon diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0a6d1ba196f..d9800eec1ac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -260,7 +260,7 @@ jobs: - name: "Downgrade PHPUnit with Paratest" shell: bash - run: "composer require --dev phpbench/phpbench:^1.2.15 phpunit/phpunit:^9.6 brianium/paratest:^6.5 symfony/console:^5.4 symfony/process:^5.4 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs --working-dir=tests" + run: "composer require --dev phpunit/phpunit:^9.6 brianium/paratest:^6.5 symfony/console:^5.4 symfony/process:^5.4 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs --working-dir=tests" - uses: ./.github/actions/downgrade-code with: diff --git a/.gitignore b/.gitignore index e240992ce18..2f3dfca6e40 100644 --- a/.gitignore +++ b/.gitignore @@ -11,10 +11,8 @@ /tests/tmp /tests/.phpunit.result.cache /tests/PHPStan/Reflection/data/golden/ -/tests/bench/storage/local-baseline.xml tmp/.memory_limit e2e/bashunit -/.phpbench /.amp/portals/ /e2e/turbo-arena/tmp /e2e/fork-unsafe-extension/tmp diff --git a/CLAUDE.md b/CLAUDE.md index 7766526269e..6bfbaa5d77f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -294,6 +294,7 @@ New introduced classes in these categories should be shadowed with a native C++ - **Always verify the test fails before the fix.** Stating "tests pass" is not enough — that only proves the fix doesn't break things. `git stash` the source change, run the test, see it fail for the right reason; `git stash pop`, run again, see it pass. Only then is the fix confirmed. The `regression-test` skill bakes this in for bug-issue workflows. - **Reproducers come from the OP's playground link** in the issue body, not from later comments. Copy the snippet as close to verbatim as practical. - **Integration tests**: `AnalyserIntegrationTest` runs full analysis on test files and checks error output. +- **Benchmarks**: every file in `tests/bench/data` is analysed in the Benchmarks workflow under CodSpeed's CPU simulation, which counts instructions instead of measuring time, so the numbers are the same on every runner. `tests/bench/measure.php` boots PHPStan and analyses a warm-up file first, so only the analysis of the benchmark file itself is counted. `tests/bench/compare.php` compares them against `tests/bench/expected.neon`; when a change moves them on purpose or adds a benchmark, replace that file with the workflow's `benchmark-measured` artifact. ### Adding support for new PHP versions @@ -311,7 +312,7 @@ Recent work on PHP 8.5 support shows the pattern: - **One fix, one commit (or one PR)**. Bench scripts and tests for a fix go in the same commit as the source change, not separately. - **Commit messages and PR titles describe the change, not the bug they close.** Prefer "Do not subtract TemplateType from TemplateType" over "Fix #14459: type subtraction". Issue closure goes in the PR body via `Closes https://github.com/phpstan/phpstan/issues/`. -- **Standard reproducer command**: `bin/phpstan analyse -l 8 --debug` (add `-vvv` for hangs and infinite loops). Files under `tests/bench/data` are not directly runnable as PHPStan inputs — copy the reproducing snippet into a `test.php` at the repo root first, then analyse that. +- **Standard reproducer command**: `bin/phpstan analyse -l 8 --debug` (add `-vvv` for hangs and infinite loops). Files under `tests/bench/data` are excluded by the default config — analyse them with the benchmarks' own config: `bin/phpstan analyse -c tests/bench/phpstan.neon --debug `. - **Debug helpers in analysed code**: `\PHPStan\dumpType($expr)` prints the inferred type of an expression at that point; `\PHPStan\debugScope()` prints the current scope. Inside `NodeScopeResolver` itself, `var_dump($scope->debug())` is the canonical inspection point. Reach for these before guessing what PHPStan is doing. - **When the user says "I don't understand why X is needed"** about existing code, treat it as a request to investigate whether X is actually needed — not a request to defend the status quo. The answer is often that two paths can be unified or one block deleted. diff --git a/Makefile b/Makefile index c6a887d6ca3..15e63882923 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ lint: XDEBUG_MODE=off php vendor/bin/parallel-lint --colors -e php,php3,php4,php5,phtml,phpt,fixed \ --exclude tests/PHPStan/Analyser/data \ --exclude tests/bench/data \ - --exclude tests/bench/RegressionBench.php \ + --exclude tests/bench/Warmup.php \ --exclude tests/PHPStan/Analyser/nsrt \ --exclude tests/PHPStan/Rules/Methods/data \ --exclude tests/PHPStan/Rules/Functions/data \ @@ -212,15 +212,6 @@ infection: --ignore-msi-with-no-mutations \ --logger-text=php://stdout -.PHONY: phpbench -phpbench: - @find tests/bench/storage/local-baseline.xml -mtime -60m | grep . || (echo "PHPBench baseline file does not exist or is too old. Regenerate it using 'make phpbench-baseline'." && exit 1) - XDEBUG_MODE=off tests/vendor/bin/phpbench run --file=tests/bench/storage/local-baseline.xml --report=my-report - -.PHONY: phpbench-baseline -phpbench-baseline: - XDEBUG_MODE=off tests/vendor/bin/phpbench run --dump-file=tests/bench/storage/local-baseline.xml - download-coverage: rm -rf tmp/coverage-download mkdir tmp/coverage-download diff --git a/build/downgrade.php b/build/downgrade.php index 437bfcf1cc8..7c77aad7d0f 100644 --- a/build/downgrade.php +++ b/build/downgrade.php @@ -6,6 +6,7 @@ __DIR__ . '/../build/PHPStan', __DIR__ . '/../src', __DIR__ . '/../tests/PHPStan', + __DIR__ . '/../tests/bench', __DIR__ . '/../tests/e2e', ], 'excludePaths' => [ diff --git a/build/ignore-by-php-version.neon.php b/build/ignore-by-php-version.neon.php index debc6ba56c9..77c8caf7c5e 100644 --- a/build/ignore-by-php-version.neon.php +++ b/build/ignore-by-php-version.neon.php @@ -27,6 +27,7 @@ if (PHP_VERSION_ID < 80200) { $includes[] = __DIR__ . '/randomizer.neon'; + $includes[] = __DIR__ . '/memory-reset-peak-usage.neon'; } if (PHP_VERSION_ID >= 80000) { diff --git a/build/memory-reset-peak-usage.neon b/build/memory-reset-peak-usage.neon new file mode 100644 index 00000000000..e7591bd6c9b --- /dev/null +++ b/build/memory-reset-peak-usage.neon @@ -0,0 +1,8 @@ +parameters: + ignoreErrors: + - + # tests/bench/measure.php runs only on the PHP 8.5 of the Benchmarks workflow + message: '#^(Used function|Function) memory_reset_peak_usage not found\.$#' + identifier: function.notFound + count: 2 + path: ../tests/bench/measure.php diff --git a/build/phpstan.neon b/build/phpstan.neon index 7bde486ad9f..68f45008c4e 100644 --- a/build/phpstan.neon +++ b/build/phpstan.neon @@ -28,12 +28,12 @@ parameters: reportPossiblyNonexistentConstantArrayOffset: true excludePaths: - ../tests/*/data/* + - ../tests/bench/Warmup.php - ../tests/tmp/* - ../tests/vendor/* - ../tests/PHPStan/Analyser/nsrt/* - ../tests/PHPStan/Analyser/traits/* - ../tests/notAutoloaded/* - - ../tests/bench/* - ../tests/PHPStan/Reflection/CircularReferenceClassReflectionTest.php - ../tests/PHPStan/Reflection/UnionTypesTest.php - ../tests/PHPStan/Reflection/MixedTypeTest.php @@ -90,6 +90,7 @@ parameters: paths: - ../src/File/FsEventsFileMonitor.php - ../src/File/InotifyFileMonitor.php + - ../tests/bench/measure.php - identifier: shipmonk.deadMethod message: '#^Unused .*?::__construct#' # likely used in DIC diff --git a/phpbench.json b/phpbench.json deleted file mode 100644 index c8c4a6da29e..00000000000 --- a/phpbench.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "./tests/vendor/phpbench/phpbench/phpbench.schema.json", - "runner.bootstrap": "vendor/autoload.php", - "runner.path": "tests/bench", - "runner.file_pattern": "*Bench.php", - "storage.xml_storage_path": "tests/bench/storage", - "report.generators": { - "my-report": { - "extends": "aggregate-preview", - "title": null, - "description": null, - "break": [], - "cols": { - "set": null, - "mem_peak": null, - "mode": null, - "rstdev": null, - "diff": null - } - } - } -} diff --git a/tests/PHPStan/Benchmark/BenchmarkComparatorTest.php b/tests/PHPStan/Benchmark/BenchmarkComparatorTest.php new file mode 100644 index 00000000000..1c6dfc863c2 --- /dev/null +++ b/tests/PHPStan/Benchmark/BenchmarkComparatorTest.php @@ -0,0 +1,79 @@ + ['instructions' => 1_000_000_000, 'estimatedCycles' => 2_000_000_000, 'peakMemory' => 100_000_000], + 'fast.php' => ['instructions' => 1_000_000_000, 'estimatedCycles' => 2_000_000_000, 'peakMemory' => 100_000_000], + 'removed.php' => ['instructions' => 500_000_000], + ]; + $actual = [ + 'slow.php' => [ + // exactly at the 5 % tolerance + 'instructions' => 1_050_000_000, + // just above it + 'estimatedCycles' => 2_102_000_000, + // above the instructions' tolerance but within the memory one + 'peakMemory' => 108_000_000, + ], + 'fast.php' => ['instructions' => 951_000_000, 'estimatedCycles' => 1_898_000_000, 'peakMemory' => 89_000_000], + 'added.php' => ['instructions' => 700], + ]; + + $this->assertSame([ + ['benchmark' => 'added.php', 'metric' => 'instructions', 'expected' => null, 'actual' => 700, 'change' => null, 'allowedChange' => null, 'status' => 'not expected'], + ['benchmark' => 'fast.php', 'metric' => 'instructions', 'expected' => 1_000_000_000, 'actual' => 951_000_000, 'change' => -0.049, 'allowedChange' => 0.05, 'status' => 'ok'], + ['benchmark' => 'fast.php', 'metric' => 'estimatedCycles', 'expected' => 2_000_000_000, 'actual' => 1_898_000_000, 'change' => -0.051, 'allowedChange' => 0.05, 'status' => 'faster'], + ['benchmark' => 'fast.php', 'metric' => 'peakMemory', 'expected' => 100_000_000, 'actual' => 89_000_000, 'change' => -0.11, 'allowedChange' => 0.1, 'status' => 'faster'], + ['benchmark' => 'removed.php', 'metric' => 'instructions', 'expected' => 500_000_000, 'actual' => null, 'change' => null, 'allowedChange' => null, 'status' => 'not measured'], + ['benchmark' => 'slow.php', 'metric' => 'instructions', 'expected' => 1_000_000_000, 'actual' => 1_050_000_000, 'change' => 0.05, 'allowedChange' => 0.05, 'status' => 'ok'], + ['benchmark' => 'slow.php', 'metric' => 'estimatedCycles', 'expected' => 2_000_000_000, 'actual' => 2_102_000_000, 'change' => 0.051, 'allowedChange' => 0.05, 'status' => 'slower'], + ['benchmark' => 'slow.php', 'metric' => 'peakMemory', 'expected' => 100_000_000, 'actual' => 108_000_000, 'change' => 0.08, 'allowedChange' => 0.1, 'status' => 'ok'], + ], BenchmarkComparator::compare($expected, $actual)); + } + + public function testSmallBenchmarkIsAllowedTheMinimumDifference(): void + { + $minimum = BenchmarkComparator::MINIMUM_ALLOWED_DIFFERENCES['instructions']; + // 5 % of it is far below the minimum, so the minimum applies + $expected = ['small.php' => ['instructions' => $minimum * 2]]; + + $results = BenchmarkComparator::compare($expected, ['small.php' => ['instructions' => $minimum * 3]]); + $this->assertSame(0.5, $results[0]['allowedChange']); + $this->assertSame('ok', $results[0]['status']); + + $results = BenchmarkComparator::compare($expected, ['small.php' => ['instructions' => $minimum * 3 + 1]]); + $this->assertSame('slower', $results[0]['status']); + + // once 5 % is more than the minimum, it applies instead + $large = $minimum * 100; + $results = BenchmarkComparator::compare(['large.php' => ['instructions' => $large]], ['large.php' => ['instructions' => $large + $minimum * 2]]); + $this->assertSame(0.05, $results[0]['allowedChange']); + $this->assertSame('ok', $results[0]['status']); + $results = BenchmarkComparator::compare(['large.php' => ['instructions' => $large]], ['large.php' => ['instructions' => $large + $minimum * 6]]); + $this->assertSame('slower', $results[0]['status']); + } + + public function testFails(): void + { + $expected = ['a.php' => ['instructions' => 1_000_000_000, 'peakMemory' => 100_000_000]]; + + $this->assertFalse(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, ['a.php' => ['instructions' => 1_050_000_000, 'peakMemory' => 110_000_000]]))); + $this->assertTrue(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, ['a.php' => ['instructions' => 1_060_000_000, 'peakMemory' => 100_000_000]]))); + $this->assertTrue(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, ['a.php' => ['instructions' => 1_000_000_000, 'peakMemory' => 111_000_000]]))); + $this->assertTrue(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, []))); + // faster and new benchmarks only ask for expected.neon to be updated + $this->assertFalse(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, [ + 'a.php' => ['instructions' => 500_000_000, 'peakMemory' => 50_000_000], + 'b.php' => ['instructions' => 100], + ]))); + } + +} diff --git a/tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php b/tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php new file mode 100644 index 00000000000..08922c5ffba --- /dev/null +++ b/tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php @@ -0,0 +1,81 @@ +assertSame([ + 'instructions' => 5_605_163_255, + // 5605163255 + 10 * (32176288 + 34788426 + 12883708) + 100 * (89670 + 1339330 + 2320891) + 'estimatedCycles' => 6_778_636_575, + ], BenchmarkProfileParser::parseCallgrindOutput(self::PROFILE, 'tests/bench/data/bug-10147.php')); + } + + public function testParseCallgrindOutputOfOtherBenchmark(): void + { + $this->assertNull(BenchmarkProfileParser::parseCallgrindOutput(self::PROFILE, 'tests/bench/data/bug-1014.php')); + } + + public function testParseCallgrindOutputOfProcessWithoutMeasurement(): void + { + $contents = <<<'CALLGRIND' + # callgrind format + cmd: /usr/bin/stty -a + part: 1 + desc: Trigger: Program termination + events: Ir Dr Dw I1mr D1mr D1mw ILmr DLmr DLmw + totals: 21618 5618 2229 497 177 70 491 177 70 + CALLGRIND; + + $this->assertNull(BenchmarkProfileParser::parseCallgrindOutput($contents, 'tests/bench/data/bug-10147.php')); + } + +} diff --git a/tests/PHPStan/Benchmark/BenchmarkReportTest.php b/tests/PHPStan/Benchmark/BenchmarkReportTest.php new file mode 100644 index 00000000000..c80ebd9a701 --- /dev/null +++ b/tests/PHPStan/Benchmark/BenchmarkReportTest.php @@ -0,0 +1,52 @@ + ['instructions' => 1_000_000_000, 'peakMemory' => 100 * 1024 * 1024], + 'b-mixed.php' => ['instructions' => 2_000_000_000, 'peakMemory' => 100 * 1024 * 1024], + 'c-faster.php' => ['instructions' => 2_000_000_000], + 'e-small.php' => ['instructions' => 10_000_000], + ], + [ + 'a-ok.php' => ['instructions' => 1_010_000_000, 'peakMemory' => 100 * 1024 * 1024], + // slower in instructions, lower in memory: listed once, as slower, with both metrics + 'b-mixed.php' => ['instructions' => 2_500_000_000, 'peakMemory' => 80 * 1024 * 1024], + 'c-faster.php' => ['instructions' => 1_000_000_000], + 'd-new.php' => ['instructions' => 1], + // 30 % more, but within the minimum allowed difference + 'e-small.php' => ['instructions' => 13_000_000], + ], + ); + + $this->assertSame( + "::error file=tests/bench/data/b-mixed.php,line=1,title=b-mixed.php is slower than expected::instructions +25.0 %25 (2.50 G, expected 2.00 G, allowed ±5 %25)%0ApeakMemory -20.0 %25 (80.0 MB, expected 100.0 MB, allowed ±10 %25)\n" + . "::warning file=tests/bench/data/d-new.php,line=1,title=d-new.php has no expected numbers::expected.neon has no numbers for these yet. This does not fail the build, but accept the new numbers.\n" + . "::warning file=tests/bench/data/c-faster.php,line=1,title=c-faster.php is faster than expected::instructions -50.0 %25 (1.00 G, expected 2.00 G, allowed ±5 %25)\n", + BenchmarkReport::renderAnnotations($results), + ); + } + + public function testRenderTerminalWhenAllMatch(): void + { + $results = BenchmarkComparator::compare( + ['a.php' => ['instructions' => 1_000_000_000], 'b.php' => ['instructions' => 1_000_000_000]], + ['a.php' => ['instructions' => 1_010_000_000], 'b.php' => ['instructions' => 990_000_000]], + ); + + $this->assertSame( + "Benchmarks passed: all 2 match tests/bench/expected.neon\n" + . "Allowed difference from tests/bench/expected.neon: ±5 % instructions (at least 25 M), ±5 % estimatedCycles (at least 35 M), ±10 % peakMemory.\n", + BenchmarkReport::renderTerminal($results, 'accept', false), + ); + } + +} diff --git a/tests/PHPStan/Benchmark/BenchmarkShardsTest.php b/tests/PHPStan/Benchmark/BenchmarkShardsTest.php new file mode 100644 index 00000000000..b98c49c844f --- /dev/null +++ b/tests/PHPStan/Benchmark/BenchmarkShardsTest.php @@ -0,0 +1,36 @@ + 7, 'a.php' => 10, 'e.php' => 5, 'b.php' => 8, 'd.php' => 6, 'f.php' => 4, 'g.php' => 2]; + + $this->assertSame([ + ['a.php', 'd.php', 'f.php', 'g.php'], + ['b.php', 'c.php', 'e.php'], + ], BenchmarkShards::split($costs, 2)); + } + + public function testSplitBreaksTiesByName(): void + { + $this->assertSame([ + ['a.php', 'd.php'], + ['b.php'], + ['c.php'], + ], BenchmarkShards::split(['d.php' => 1, 'c.php' => 1, 'b.php' => 1, 'a.php' => 1], 3)); + } + + public function testSplitIntoMoreShardsThanBenchmarks(): void + { + $this->assertSame([['a.php'], ['b.php']], BenchmarkShards::split(['a.php' => 3, 'b.php' => 1], 4)); + } + +} diff --git a/tests/bench/BenchCase.php b/tests/bench/BenchCase.php deleted file mode 100644 index 779718d72a8..00000000000 --- a/tests/bench/BenchCase.php +++ /dev/null @@ -1,39 +0,0 @@ - - */ - protected function runAnalyse(string $file, ?array $allAnalysedFiles = null): array - { - $file = self::getFileHelper()->normalizePath($file); - - $analyser = self::getContainer()->getByType(Analyser::class); - $finalizer = self::getContainer()->getByType(AnalyserResultFinalizer::class); - return $finalizer->finalize( - $analyser->analyse([$file], null, null, true, $allAnalysedFiles), - false, - true, - )->getErrors(); - } - -} diff --git a/tests/bench/BenchmarkComparator.php b/tests/bench/BenchmarkComparator.php new file mode 100644 index 00000000000..93388de5615 --- /dev/null +++ b/tests/bench/BenchmarkComparator.php @@ -0,0 +1,120 @@ + 0.05, + 'estimatedCycles' => 0.05, + 'peakMemory' => 0.10, + ]; + + /** Allowed difference from the expected value however small the benchmark is, per metric */ + public const MINIMUM_ALLOWED_DIFFERENCES = [ + 'instructions' => 25_000_000, + 'estimatedCycles' => 35_000_000, + 'peakMemory' => 0, + ]; + + /** + * @param array> $expected metric values keyed by benchmark file name and metric name + * @param array> $actual the same shape, as measured + * @return list, expected: int|null, actual: int|null, change: float|null, allowedChange: float|null, status: self::STATUS_*}> sorted by benchmark, then in the order of TOLERANCES + */ + public static function compare(array $expected, array $actual): array + { + $benchmarks = array_unique([...array_keys($expected), ...array_keys($actual)]); + sort($benchmarks, SORT_STRING); + + $results = []; + foreach ($benchmarks as $benchmark) { + foreach (self::TOLERANCES as $metric => $tolerance) { + $expectedValue = $expected[$benchmark][$metric] ?? null; + $actualValue = $actual[$benchmark][$metric] ?? null; + if ($expectedValue === null && $actualValue === null) { + continue; + } + + $change = null; + $allowedChange = null; + if ($expectedValue === null) { + $status = self::STATUS_NOT_EXPECTED; + } elseif ($actualValue === null) { + $status = self::STATUS_NOT_MEASURED; + } else { + $change = ($actualValue - $expectedValue) / (float) $expectedValue; + $allowedChange = max($tolerance, self::MINIMUM_ALLOWED_DIFFERENCES[$metric] / (float) $expectedValue); + if (abs($change) <= $allowedChange) { + $status = self::STATUS_OK; + } elseif ($change > 0) { + $status = self::STATUS_SLOWER; + } else { + $status = self::STATUS_FASTER; + } + } + + $results[] = [ + 'benchmark' => $benchmark, + 'metric' => $metric, + 'expected' => $expectedValue, + 'actual' => $actualValue, + 'change' => $change, + 'allowedChange' => $allowedChange, + 'status' => $status, + ]; + } + } + + return $results; + } + + /** + * @param list $results from compare() + */ + public static function fails(array $results): bool + { + foreach ($results as $result) { + if (in_array($result['status'], self::FAILING_STATUSES, true)) { + return true; + } + } + + return false; + } + +} diff --git a/tests/bench/BenchmarkProfileParser.php b/tests/bench/BenchmarkProfileParser.php new file mode 100644 index 00000000000..9141d6ffbcf --- /dev/null +++ b/tests/bench/BenchmarkProfileParser.php @@ -0,0 +1,70 @@ + $instructions, + 'estimatedCycles' => $instructions + 10 * $firstLevelMisses + 100 * $lastLevelMisses, + ]; + } + } + + return null; + } + + /** + * @return list + */ + private static function splitFields(string $fields): array + { + return explode(' ', trim($fields)); + } + +} diff --git a/tests/bench/BenchmarkReport.php b/tests/bench/BenchmarkReport.php new file mode 100644 index 00000000000..92ea39818e2 --- /dev/null +++ b/tests/bench/BenchmarkReport.php @@ -0,0 +1,390 @@ +, expected: int|null, actual: int|null, change: float|null, allowedChange: float|null, status: BenchmarkComparator::STATUS_*} + */ +final class BenchmarkReport +{ + + /** The sections in the order they are shown, the first one a benchmark has a metric in wins */ + private const SECTIONS = [ + BenchmarkComparator::STATUS_SLOWER => [ + 'title' => 'Slower than expected', + 'alert' => 'CAUTION', + 'color' => '31', + 'annotation' => 'error', + 'annotationTitle' => '%s is slower than expected', + 'text' => 'This change makes PHPStan do more work on these files. Find out why before merging. If it is the price of a fix, accept the new numbers.', + ], + BenchmarkComparator::STATUS_NOT_MEASURED => [ + 'title' => 'Not measured', + 'alert' => 'WARNING', + 'color' => '33', + 'annotation' => 'error', + 'annotationTitle' => '%s was not measured', + 'text' => 'These benchmarks failed or timed out, see the log of their shard. A benchmark removed from data/ also has to be removed from expected.neon.', + ], + BenchmarkComparator::STATUS_NOT_EXPECTED => [ + 'title' => 'New benchmarks', + 'alert' => 'NOTE', + 'color' => '34', + 'annotation' => 'warning', + 'annotationTitle' => '%s has no expected numbers', + 'text' => 'expected.neon has no numbers for these yet. This does not fail the build, but accept the new numbers.', + ], + BenchmarkComparator::STATUS_FASTER => [ + 'title' => 'Faster than expected', + 'alert' => 'TIP', + 'color' => '32', + 'annotation' => 'warning', + 'annotationTitle' => '%s is faster than expected', + 'text' => 'Nice! This does not fail the build, but accept the new numbers, so a later slowdown back to the old ones does not go unnoticed.', + ], + ]; + + private const HEADER = ['Benchmark', 'Metric', 'Expected', 'Measured', 'Change', 'Allowed']; + + /** + * @param list $results + * @param string $acceptCommand shell commands that replace expected.neon with the measured numbers + */ + public static function renderTerminal(array $results, string $acceptCommand, bool $colors): string + { + $style = static fn (string $code, string $text): string => $colors ? "\e[" . $code . 'm' . $text . "\e[0m" : $text; + $sections = self::groupBySection($results); + + $output = $style('1', self::headline($results, $sections)) . "\n"; + $output .= self::toleranceLine() . "\n"; + foreach ($sections as $status => $rows) { + $section = self::SECTIONS[$status]; + $output .= "\n" . $style('1;' . $section['color'], $section['title']) . "\n"; + $output .= $section['text'] . "\n\n"; + $output .= self::renderTextTable(self::tableRows($rows), ' '); + } + if ($sections === []) { + return $output; + } + + $output .= "\n" . $style('1', 'Reproduce locally') . ', on this branch and on the base branch:' . "\n"; + $output .= ' ' . self::reproduceCommand($sections) . "\n"; + $output .= "\n" . $style('1', 'Accept the new numbers') . "\n"; + $output .= ' ' . str_replace("\n", "\n ", $acceptCommand) . "\n"; + + return $output; + } + + /** + * Every benchmark with its metrics, for the collapsed part of the log. + * + * @param list $results + */ + public static function renderTerminalTable(array $results): string + { + return self::renderTextTable([ + [...self::HEADER, 'Status'], + ...array_map(static fn (array $result): array => [...self::tableRow($result), $result['status']], $results), + ], ''); + } + + /** + * @param list $results + */ + public static function renderMarkdown(array $results, string $acceptCommand): string + { + $sections = self::groupBySection($results); + + $output = '## ' . self::headline($results, $sections) . "\n\n"; + $output .= self::toleranceLine() . "\n"; + foreach ($sections as $status => $rows) { + $section = self::SECTIONS[$status]; + $output .= sprintf("\n> [!%s]\n> **%s**\n> %s\n\n", $section['alert'], $section['title'], $section['text']); + $output .= self::renderMarkdownTable(self::tableRows($rows)); + } + if ($sections !== []) { + $output .= "\n### Reproduce locally\n\nOn this branch and on the base branch:\n\n```\n" . self::reproduceCommand($sections) . "\n```\n"; + $output .= "\n### Accept the new numbers\n\n```\n" . $acceptCommand . "\n```\n"; + } + + $output .= "\n
All benchmarks\n\n"; + $output .= self::renderMarkdownTable([ + [...self::HEADER, 'Status'], + ...array_map(static fn (array $result): array => [...self::tableRow($result), $result['status']], $results), + ]); + + return $output . "\n
\n"; + } + + /** + * GitHub Actions workflow commands that annotate the benchmark files, one per benchmark. + * + * @param list $results + */ + public static function renderAnnotations(array $results): string + { + $output = ''; + foreach (self::groupBySection($results) as $status => $rows) { + $byBenchmark = []; + foreach ($rows as $row) { + $byBenchmark[$row['benchmark']][] = $row; + } + foreach ($byBenchmark as $benchmark => $benchmarkRows) { + $changes = []; + foreach ($benchmarkRows as $row) { + if ($row['change'] === null) { + continue; + } + $changes[] = sprintf( + '%s %s (%s, expected %s, allowed ±%s %%)', + $row['metric'], + self::formatChange($row['change']), + self::formatShort($row['metric'], $row['actual']), + self::formatShort($row['metric'], $row['expected']), + self::formatPercent($row['allowedChange'] ?? 0.0), + ); + } + $output .= sprintf( + "::%s file=tests/bench/data/%s,line=1,title=%s::%s\n", + self::SECTIONS[$status]['annotation'], + $benchmark, + self::escapeAnnotation(sprintf(self::SECTIONS[$status]['annotationTitle'], $benchmark)), + self::escapeAnnotation($changes === [] ? self::SECTIONS[$status]['text'] : implode("\n", $changes)), + ); + } + } + + return $output; + } + + /** + * @param list $results + * @return array, non-empty-list> only the sections with a benchmark, in the order of SECTIONS + */ + private static function groupBySection(array $results): array + { + $statusesByBenchmark = []; + foreach ($results as $result) { + $statusesByBenchmark[$result['benchmark']][$result['status']] = true; + } + + $sections = []; + foreach ($results as $result) { + if ($result['status'] === BenchmarkComparator::STATUS_OK) { + continue; + } + foreach (array_keys(self::SECTIONS) as $status) { + if (isset($statusesByBenchmark[$result['benchmark']][$status])) { + $sections[$status][] = $result; + break; + } + } + } + + $ordered = []; + foreach (array_keys(self::SECTIONS) as $status) { + if (!isset($sections[$status])) { + continue; + } + + $ordered[$status] = $sections[$status]; + } + + return $ordered; + } + + /** + * @param list $results + * @param array, non-empty-list> $sections + */ + private static function headline(array $results, array $sections): string + { + $benchmarkCounts = []; + foreach ($sections as $status => $rows) { + $benchmarkCounts[$status] = count(array_unique(array_map(static fn (array $row): string => $row['benchmark'], $rows))); + } + $allBenchmarks = count(array_unique(array_map(static fn (array $result): string => $result['benchmark'], $results))); + $unchanged = $allBenchmarks - array_sum($benchmarkCounts); + if ($benchmarkCounts === []) { + return sprintf('Benchmarks passed: all %d match tests/bench/expected.neon', $allBenchmarks); + } + + $parts = []; + foreach ($benchmarkCounts as $status => $benchmarkCount) { + $parts[] = sprintf('%d %s', $benchmarkCount, $status === BenchmarkComparator::STATUS_NOT_EXPECTED ? 'new' : $status); + } + $parts[] = sprintf('%d unchanged', $unchanged); + + return (BenchmarkComparator::fails($results) ? 'Benchmarks failed: ' : 'Benchmarks passed: ') . implode(', ', $parts); + } + + private static function toleranceLine(): string + { + $tolerances = []; + foreach (BenchmarkComparator::TOLERANCES as $metric => $tolerance) { + $minimum = BenchmarkComparator::MINIMUM_ALLOWED_DIFFERENCES[$metric]; + $tolerances[] = $minimum > 0 + ? sprintf('±%d %% %s (at least %s)', (int) round($tolerance * 100), $metric, self::formatShort($metric, $minimum)) + : sprintf('±%d %% %s', (int) round($tolerance * 100), $metric); + } + + return 'Allowed difference from tests/bench/expected.neon: ' . implode(', ', $tolerances) . '.'; + } + + /** + * @param array, non-empty-list> $sections + */ + private static function reproduceCommand(array $sections): string + { + $worst = null; + foreach ($sections[array_key_first($sections) ?? BenchmarkComparator::STATUS_SLOWER] ?? [] as $row) { + if ($worst !== null && abs($row['change'] ?? 0.0) <= abs($worst['change'] ?? 0.0)) { + continue; + } + + $worst = $row; + } + if ($worst === null) { + return ''; + } + + return 'bin/phpstan analyse -c tests/bench/phpstan.neon --debug tests/bench/data/' . $worst['benchmark']; + } + + /** + * @param list $rows + * @return non-empty-list> + */ + private static function tableRows(array $rows): array + { + return [self::HEADER, ...array_map(static fn (array $row): array => self::tableRow($row), $rows)]; + } + + /** + * @param Result $result + * @return list + */ + private static function tableRow(array $result): array + { + return [ + $result['benchmark'], + $result['metric'], + self::formatValue($result['metric'], $result['expected']), + self::formatValue($result['metric'], $result['actual']), + $result['change'] === null ? '-' : self::formatChange($result['change']), + $result['allowedChange'] === null ? '-' : '±' . self::formatPercent($result['allowedChange']) . ' %', + ]; + } + + /** + * Left-aligns the first two columns and right-aligns the numbers. + * + * @param list> $rows the first one is the header + */ + private static function renderTextTable(array $rows, string $indent): string + { + $widths = []; + foreach ($rows as $row) { + foreach ($row as $i => $cell) { + $widths[$i] = max($widths[$i] ?? 0, mb_strlen($cell)); + } + } + + $lines = []; + foreach ($rows as $row) { + $cells = []; + foreach ($row as $i => $cell) { + $cells[] = mb_str_pad($cell, $widths[$i], ' ', $i < 2 || $i > 5 ? STR_PAD_RIGHT : STR_PAD_LEFT); + } + $lines[] = rtrim($indent . implode(' ', $cells)); + } + + return implode("\n", $lines) . "\n"; + } + + /** + * @param non-empty-list> $rows the first one is the header + */ + private static function renderMarkdownTable(array $rows): string + { + $header = array_shift($rows); + $output = '| ' . implode(' | ', $header) . " |\n"; + $output .= '|' . implode('|', array_map(static fn (int $i): string => $i < 2 || $i > 5 ? '---' : '--:', array_keys($header))) . "|\n"; + foreach ($rows as $row) { + $row[0] = '`' . $row[0] . '`'; + $output .= '| ' . implode(' | ', $row) . " |\n"; + } + + return $output; + } + + private static function formatValue(string $metric, ?int $value): string + { + if ($value === null) { + return '-'; + } + if ($metric === 'peakMemory') { + return number_format($value / 1024 / 1024, 1) . ' MB'; + } + + return number_format($value); + } + + private static function formatShort(string $metric, ?int $value): string + { + if ($value === null || $metric === 'peakMemory') { + return self::formatValue($metric, $value); + } + + if ($value < 1_000_000_000) { + return number_format($value / 1e6) . ' M'; + } + + return number_format($value / 1e9, 2) . ' G'; + } + + private static function formatPercent(float $fraction): string + { + $percent = $fraction * 100; + + return $percent >= 10 || $percent === round($percent) ? (string) (int) round($percent) : sprintf('%.1f', $percent); + } + + private static function formatChange(float $change): string + { + return sprintf('%+.1f %%', $change * 100); + } + + /** https://github.com/actions/toolkit/blob/main/packages/core/src/command.ts */ + private static function escapeAnnotation(string $text): string + { + return str_replace(['%', "\r", "\n"], ['%25', '%0D', '%0A'], $text); + } + +} diff --git a/tests/bench/BenchmarkShards.php b/tests/bench/BenchmarkShards.php new file mode 100644 index 00000000000..c3a2426aaf5 --- /dev/null +++ b/tests/bench/BenchmarkShards.php @@ -0,0 +1,52 @@ + $costs expected cost keyed by benchmark + * @return list> benchmarks of each shard from the most expensive one, without empty shards + */ + public static function split(array $costs, int $count): array + { + // ties are broken by name, so the shards stay the same while the costs do + ksort($costs, SORT_STRING); + arsort($costs, SORT_NUMERIC); + + $shards = []; + $totals = []; + foreach (range(0, $count - 1) as $shard) { + $shards[$shard] = []; + $totals[$shard] = 0; + } + + foreach ($costs as $benchmark => $cost) { + $cheapestShard = array_keys($totals, min($totals), true)[0]; + $shards[$cheapestShard][] = $benchmark; + $totals[$cheapestShard] += $cost; + } + + return array_values(array_filter($shards, static fn (array $benchmarks): bool => count($benchmarks) > 0)); + } + +} diff --git a/tests/bench/RegressionBench.php b/tests/bench/RegressionBench.php deleted file mode 100644 index bef98a0789f..00000000000 --- a/tests/bench/RegressionBench.php +++ /dev/null @@ -1,51 +0,0 @@ -= 100 milliseconds and mode(baseline.time.avg) < 500 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 25%) - or (mode(baseline.time.avg) >= 500 milliseconds and mode(baseline.time.avg) < 2000 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 20%) - or (mode(baseline.time.avg) >= 2000 milliseconds and mode(variant.time.avg) < mode(baseline.time.avg) +/- 10%) -')] -class RegressionBench extends BenchCase -{ - - /** - * @param array{string} $params - */ - #[Bench\ParamProviders(['provideFiles'])] - public function benchRunAnalyse(array $params): void - { - $this->runAnalyse($params[0]); - } - - /** - * @return iterable - */ - public function provideFiles(): iterable - { - yield from self::findTestDataFilesFromDirectory(__DIR__ . '/data'); - } - - private static function findTestDataFilesFromDirectory(string $directory): array - { - $finder = new Finder(); - $finder->followLinks(); - $finder->sortByName(true); - $files = []; - foreach ($finder->files()->name('*.php')->in($directory) as $fileInfo) { - $files[$fileInfo->getBasename()] = [$fileInfo->getPathname()]; - } - - return $files; - } - -} diff --git a/tests/bench/Warmup.php b/tests/bench/Warmup.php new file mode 100644 index 00000000000..82fb96189fa --- /dev/null +++ b/tests/bench/Warmup.php @@ -0,0 +1,80 @@ +> */ + private array $values = []; + + public function __construct(private readonly string $name) + { + } + + /** + * @param list $numbers + */ + public function add(string $key, array $numbers): void + { + foreach ($numbers as $number) { + if ($number < 0 || in_array($number, $this->values[$key] ?? [], true)) { + continue; + } + + $this->values[$key][] = $number; + } + } + + /** + * @return array + */ + public function totals(): array + { + $totals = []; + foreach ($this->values as $key => $numbers) { + $total = 0; + foreach (array_filter($numbers, static fn (int $number): bool => $number % 2 === 0) as $number) { + $total += $number; + } + $totals[$key] = $total; + } + + return $totals; + } + + public function describe(?int $limit = null): string + { + $lines = array_map( + static fn (string $key, int $total): string => sprintf('%s: %d', $key, $total), + array_keys($this->totals()), + $this->totals(), + ); + if ($limit !== null && count($lines) > $limit) { + $lines = array_slice($lines, 0, $limit); + } + + $description = match (count($lines)) { + 0 => 'empty', + 1 => 'one value', + default => 'more values', + }; + + return strlen($this->name) > 0 ? $this->name . ' ' . $description : $description; + } + +} diff --git a/tests/bench/compare.php b/tests/bench/compare.php new file mode 100644 index 00000000000..efb38321872 --- /dev/null +++ b/tests/bench/compare.php @@ -0,0 +1,78 @@ + [] + * + * The written file has the format of expected.neon, so it can replace it after an intended change. + */ + +use Nette\Neon\Neon; +use function count; +use function file_get_contents; +use function file_put_contents; +use function fwrite; +use function getenv; +use function glob; +use function is_file; +use function ksort; +use function sprintf; +use function stream_isatty; +use const FILE_APPEND; +use const SORT_STRING; +use const STDERR; +use const STDOUT; + +require_once __DIR__ . '/../../vendor/autoload.php'; + +if (!isset($argv) || (count($argv) !== 2 && count($argv) !== 3)) { + fwrite(STDERR, "Usage: php tests/bench/compare.php []\n"); + exit(2); +} + +$expectedFile = __DIR__ . '/expected.neon'; +$expected = is_file($expectedFile) ? Neon::decode((string) file_get_contents($expectedFile)) : []; + +$actual = []; +foreach (glob($argv[1] . '/*.neon') ?: [] as $resultFile) { + $actual += Neon::decode((string) file_get_contents($resultFile)); +} +ksort($actual, SORT_STRING); + +if (isset($argv[2])) { + $header = "# Expected metrics of the benchmarks in data/, compared by compare.php with the tolerances in BenchmarkComparator.\n" + . "# To update it, replace it with the benchmark-measured artifact of the Benchmarks workflow.\n\n"; + file_put_contents($argv[2], $header . Neon::encode($actual, true)); +} + +$results = BenchmarkComparator::compare($expected, $actual); + +$onGitHub = getenv('GITHUB_ACTIONS') === 'true'; +$runId = getenv('GITHUB_RUN_ID'); +$repository = getenv('GITHUB_REPOSITORY'); +$acceptCommand = sprintf( + "gh run download %s%s -n benchmark-measured -D tmp/benchmark-measured\nmv tmp/benchmark-measured/expected.neon tests/bench/expected.neon", + $runId !== false && $runId !== '' ? $runId : '', + $repository !== false && $repository !== '' ? ' -R ' . $repository : '', +); + +echo BenchmarkReport::renderTerminal($results, $acceptCommand, $onGitHub || stream_isatty(STDOUT)); +echo "\n"; +echo $onGitHub ? "::group::All benchmarks\n" : "All benchmarks\n\n"; +echo BenchmarkReport::renderTerminalTable($results); +if ($onGitHub) { + echo "::endgroup::\n"; + echo BenchmarkReport::renderAnnotations($results); +} + +$stepSummaryFile = getenv('GITHUB_STEP_SUMMARY'); +if ($stepSummaryFile !== false && $stepSummaryFile !== '') { + file_put_contents($stepSummaryFile, BenchmarkReport::renderMarkdown($results, $acceptCommand), FILE_APPEND); +} + +exit(BenchmarkComparator::fails($results) ? 1 : 0); diff --git a/tests/bench/expected.neon b/tests/bench/expected.neon new file mode 100644 index 00000000000..ae73e32d2c7 --- /dev/null +++ b/tests/bench/expected.neon @@ -0,0 +1,468 @@ +# Expected metrics of the benchmarks in data/, compared by compare.php with the tolerances in BenchmarkComparator. +# To update it, replace it with the benchmark-measured artifact of the Benchmarks workflow. + +and-chain-resolve-type-blowup.php: + instructions: 3118737800 + estimatedCycles: 3396057190 + peakMemory: 46580928 + +and-chain-truthy-blowup.php: + instructions: 260817814 + estimatedCycles: 338603194 + peakMemory: 41321776 + +array-values-literal-keyed-map.php: + instructions: 3362949832 + estimatedCycles: 3505853792 + peakMemory: 42747512 + +big-constant-int-union.php: + instructions: 398891207 + estimatedCycles: 464751137 + peakMemory: 42336512 + +big-constant-string-union.php: + instructions: 1710326116 + estimatedCycles: 1931192206 + peakMemory: 42922088 + +bug-10147.php: + instructions: 11290386 + estimatedCycles: 18120896 + peakMemory: 37501872 + +bug-10538.php: + instructions: 18121402938 + estimatedCycles: 21830691058 + peakMemory: 55908888 + +bug-10772.php: + instructions: 4339252871 + estimatedCycles: 5682093521 + peakMemory: 98737480 + +bug-10979.php: + instructions: 4077060639 + estimatedCycles: 5525781109 + peakMemory: 44319104 + +bug-11263.php: + instructions: 1927527402 + estimatedCycles: 2270006532 + peakMemory: 44125752 + +bug-11283.php: + instructions: 1236834912 + estimatedCycles: 1626637602 + peakMemory: 42002416 + +bug-11297.php: + instructions: 9220268282 + estimatedCycles: 10151353832 + peakMemory: 68972408 + +bug-11913.php: + instructions: 998145804 + estimatedCycles: 1278606784 + peakMemory: 55738544 + +bug-12159.php: + instructions: 1841218978 + estimatedCycles: 2281466968 + peakMemory: 60920784 + +bug-12671.php: + instructions: 4594854456 + estimatedCycles: 5549796096 + peakMemory: 69275720 + +bug-12787.php: + instructions: 44474610 + estimatedCycles: 65742240 + peakMemory: 38185112 + +bug-12800.php: + instructions: 672274779 + estimatedCycles: 805487759 + peakMemory: 41384888 + +bug-13218.php: + instructions: 351063825 + estimatedCycles: 436627115 + peakMemory: 41070880 + +bug-13310.php: + instructions: 60495534 + estimatedCycles: 85843194 + peakMemory: 38107016 + +bug-13352.php: + instructions: 13076427844 + estimatedCycles: 17917315214 + peakMemory: 182733352 + +bug-13685.php: + instructions: 143099972 + estimatedCycles: 194935342 + peakMemory: 39006744 + +bug-1388.php: + instructions: 355936318 + estimatedCycles: 458614618 + peakMemory: 40047608 + +bug-13933.php: + instructions: 1038298881 + estimatedCycles: 1320234791 + peakMemory: 51901808 + +bug-14207-and.php: + instructions: 279494690 + estimatedCycles: 356959470 + peakMemory: 41369736 + +bug-14207.php: + instructions: 447091805 + estimatedCycles: 570791625 + peakMemory: 43156888 + +bug-14319.php: + instructions: 340674847 + estimatedCycles: 434737977 + peakMemory: 39199648 + +bug-14452.php: + instructions: 153974253 + estimatedCycles: 206422453 + peakMemory: 40142632 + +bug-14462.php: + instructions: 1192503989 + estimatedCycles: 1488009519 + peakMemory: 40067736 + +bug-1447.php: + instructions: 104050647 + estimatedCycles: 143388637 + peakMemory: 38950920 + +bug-14475.php: + instructions: 158065427 + estimatedCycles: 206349897 + peakMemory: 38957256 + +bug-14590.php: + instructions: 974467037 + estimatedCycles: 1261327667 + peakMemory: 38418920 + +bug-14624.php: + instructions: 11014859209 + estimatedCycles: 13476004149 + peakMemory: 51272776 + +bug-14674.php: + instructions: 7327393991 + estimatedCycles: 9337273351 + peakMemory: 45847744 + +bug-14869-enum.php: + instructions: 1622188111 + estimatedCycles: 1943130031 + peakMemory: 54782960 + +bug-14869.php: + instructions: 2009133975 + estimatedCycles: 2383528125 + peakMemory: 49150464 + +bug-14972-concat.php: + instructions: 7856536418 + estimatedCycles: 10942624748 + peakMemory: 184134120 + +bug-14972.php: + instructions: 9443587818 + estimatedCycles: 11145824458 + peakMemory: 141880680 + +bug-14996.php: + instructions: 9715463268 + estimatedCycles: 11967965898 + peakMemory: 46647912 + +bug-15061.php: + instructions: 3208835402 + estimatedCycles: 3641897892 + peakMemory: 41463688 + +bug-15305.php: + instructions: 3258352078 + estimatedCycles: 4060295938 + peakMemory: 39323216 + +bug-15348.php: + instructions: 8120161324 + estimatedCycles: 8838441104 + peakMemory: 46906440 + +bug-15452.php: + instructions: 1776848094 + estimatedCycles: 2375240804 + peakMemory: 69539152 + +bug-15498.php: + instructions: 395716533 + estimatedCycles: 501405533 + peakMemory: 39072416 + +bug-15591.php: + instructions: 41795437 + estimatedCycles: 62239297 + peakMemory: 37742320 + +bug-3686.php: + instructions: 61008866 + estimatedCycles: 87076096 + peakMemory: 38205816 + +bug-4300.php: + instructions: 26966115 + estimatedCycles: 41356215 + peakMemory: 37708896 + +bug-4308.php: + instructions: 27527885 + estimatedCycles: 40028715 + peakMemory: 37579584 + +bug-5081.php: + instructions: 4255601191 + estimatedCycles: 4962251191 + peakMemory: 117158552 + +bug-5231.php: + instructions: 402207359 + estimatedCycles: 505251819 + peakMemory: 41067192 + +bug-5231_2.php: + instructions: 139149226 + estimatedCycles: 191385216 + peakMemory: 39247592 + +bug-5390.php: + instructions: 20107966 + estimatedCycles: 31941626 + peakMemory: 38016120 + +bug-6265.php: + instructions: 631694840 + estimatedCycles: 831423710 + peakMemory: 41240040 + +bug-6442.php: + instructions: 35405752 + estimatedCycles: 52214732 + peakMemory: 38603360 + +bug-6936.php: + instructions: 230391547 + estimatedCycles: 304471867 + peakMemory: 40311304 + +bug-6948.php: + instructions: 246653259 + estimatedCycles: 313870599 + peakMemory: 38744808 + +bug-7140.php: + instructions: 153996491 + estimatedCycles: 209139071 + peakMemory: 38867968 + +bug-7214.php: + instructions: 52221045 + estimatedCycles: 73361875 + peakMemory: 38781464 + +bug-7581.php: + instructions: 3174182540 + estimatedCycles: 3762296570 + peakMemory: 46453880 + +bug-7637.php: + instructions: 189252511 + estimatedCycles: 239162981 + peakMemory: 39713864 + +bug-7901.php: + instructions: 5140910531 + estimatedCycles: 6698549141 + peakMemory: 93494056 + +bug-7903.php: + instructions: 7792909233 + estimatedCycles: 9795796913 + peakMemory: 50539856 + +bug-8146a.php: + instructions: 882474698 + estimatedCycles: 1123302618 + peakMemory: 41358904 + +bug-8146b.php: + instructions: 7072474770 + estimatedCycles: 9339299610 + peakMemory: 105031048 + +bug-8147.php: + instructions: 2989403361 + estimatedCycles: 3950497151 + peakMemory: 71138992 + +bug-8215.php: + instructions: 5745263279 + estimatedCycles: 7626562969 + peakMemory: 136631192 + +bug-8503.php: + instructions: 973796790 + estimatedCycles: 1258568170 + peakMemory: 45359688 + +bug-9690.php: + instructions: 396880979 + estimatedCycles: 523618309 + peakMemory: 41030152 + +callable-union-describe.php: + instructions: 19980240224 + estimatedCycles: 24734732134 + peakMemory: 64387232 + +closure-variable-invocations.php: + instructions: 3280559424 + estimatedCycles: 4470267984 + peakMemory: 76077024 + +conditional-expression-infinite-loop.php: + instructions: 53507832 + estimatedCycles: 76010282 + peakMemory: 37799152 + +enum-case-union-removal.php: + instructions: 1923127781 + estimatedCycles: 2338744131 + peakMemory: 44180760 + +enum-value-keyed-map.php: + instructions: 702961621 + estimatedCycles: 884545891 + peakMemory: 55992320 + +finite-types-optional-keys-blowup.php: + instructions: 38584310 + estimatedCycles: 50647870 + peakMemory: 37663192 + +finite-types-optional-keys-count.php: + instructions: 4903672367 + estimatedCycles: 6034352347 + peakMemory: 80668528 + +flatten-types-optional-keys-blowup.php: + instructions: 33827718 + estimatedCycles: 46106158 + peakMemory: 37732112 + +hash-key-lookup.php: + instructions: 3543036508 + estimatedCycles: 4668882598 + peakMemory: 70125440 + +implode-optional-keys-blowup.php: + instructions: 36455809 + estimatedCycles: 50241199 + peakMemory: 37768632 + +impure-call-columns.php: + instructions: 3250463482 + estimatedCycles: 4485812952 + peakMemory: 109521432 + +in-array-intersect-blowup.php: + instructions: 168142186 + estimatedCycles: 202690526 + peakMemory: 38427984 + +literal-key-union-offset-write.php: + instructions: 10711284461 + estimatedCycles: 14697676331 + peakMemory: 224807280 + +long-statement-list.php: + instructions: 4213145447 + estimatedCycles: 5758908777 + peakMemory: 73321848 + +loop-widening-optional-keys.php: + instructions: 1126314976 + estimatedCycles: 1453386996 + peakMemory: 43146400 + +loose-string-comparison.php: + instructions: 10230458586 + estimatedCycles: 12287958386 + peakMemory: 89695768 + +nullsafe-chain-walk.php: + instructions: 18582353612 + estimatedCycles: 23382482282 + peakMemory: 103604264 + +or-chain-falsey-blowup.php: + instructions: 247621514 + estimatedCycles: 314218854 + peakMemory: 41512976 + +or-chain-resolve-type-blowup.php: + instructions: 3083688735 + estimatedCycles: 3347462195 + peakMemory: 45856224 + +phpdoc-inheritance-speedup.php: + instructions: 8914065482 + estimatedCycles: 9944220742 + peakMemory: 50479896 + +phpdoc-intersection-blowup.php: + instructions: 49418634 + estimatedCycles: 65565794 + peakMemory: 37757136 + +process-called-method-infinite-loop.php: + instructions: 367000513 + estimatedCycles: 460612113 + peakMemory: 41311848 + +sealed-shape-if-chain.php: + instructions: 1635665620 + estimatedCycles: 1991942830 + peakMemory: 61502480 + +union-fast-path.php: + instructions: 818818636 + estimatedCycles: 1049785036 + peakMemory: 41353752 + +unrolled-foreach-list-guards.php: + instructions: 1749734463 + estimatedCycles: 2089656253 + peakMemory: 40596720 + +wordpress-user.php: + instructions: 2804177998 + estimatedCycles: 3555429078 + peakMemory: 47112096 + diff --git a/tests/bench/measure.php b/tests/bench/measure.php new file mode 100644 index 00000000000..08e7afb774e --- /dev/null +++ b/tests/bench/measure.php @@ -0,0 +1,138 @@ + + * + * The INSTRUMENT_HOOKS_LIBRARY environment variable points to CodSpeed's instrument-hooks + * library (https://github.com/CodSpeedHQ/instrument-hooks). Without it, the analysis runs + * unmeasured, which run.php uses to fill the caches in tmpDir before the measured run. + * + * The peak memory is the highest memory usage during the benchmark's analysis, including what + * the booted container and the warm-up keep allocated. + */ + +use FFI; +use PHPStan\Analyser\Analyser; +use PHPStan\Analyser\AnalyserResultFinalizer; +use PHPStan\Command\CommandHelper; +use Symfony\Component\Console\Input\ArrayInput; +use Symfony\Component\Console\Output\StreamOutput; +use function basename; +use function count; +use function dirname; +use function error_reporting; +use function file_put_contents; +use function fwrite; +use function gc_disable; +use function getenv; +use function getmypid; +use function memory_get_peak_usage; +use function memory_reset_peak_usage; +use function realpath; +use function sprintf; +use const E_ALL; +use const E_DEPRECATED; +use const STDERR; + +// Like bin/phpstan +error_reporting(E_ALL & ~E_DEPRECATED); +gc_disable(); + +$rootDir = dirname(__DIR__, 2); +require_once $rootDir . '/vendor/autoload.php'; + +if (!isset($argv) || count($argv) !== 4) { + fwrite(STDERR, "Usage: php -d ffi.enable=1 tests/bench/measure.php \n"); + exit(2); +} + +[, $configFile, $benchmarkFile, $peakMemoryFile] = $argv; +$benchmarkFile = realpath($benchmarkFile); +$warmupFile = __DIR__ . '/Warmup.php'; +if ($benchmarkFile === false) { + fwrite(STDERR, "The benchmark file does not exist.\n"); + exit(2); +} + +$hooks = null; +$hooksLibrary = getenv('INSTRUMENT_HOOKS_LIBRARY'); +if ($hooksLibrary !== false && $hooksLibrary !== '') { + $hooks = FFI::cdef(' + typedef struct InstrumentHooks InstrumentHooks; + InstrumentHooks *instrument_hooks_init(void); + bool instrument_hooks_is_instrumented(InstrumentHooks *); + uint8_t instrument_hooks_start_benchmark(InstrumentHooks *); + uint8_t instrument_hooks_stop_benchmark(InstrumentHooks *); + uint8_t instrument_hooks_set_executed_benchmark(InstrumentHooks *, int32_t pid, const char *uri); + uint8_t instrument_hooks_set_integration(InstrumentHooks *, const char *name, const char *version); + ', $hooksLibrary); +} + +$output = new StreamOutput(STDERR); +$inceptionResult = CommandHelper::begin( + new ArrayInput([]), + $output, + [$warmupFile, $benchmarkFile], + '-1', + null, + [$rootDir], + $configFile, + null, + null, + false, + true, + null, + null, + false, +); +// Also tells the parser which files are analysed, the others are parsed without function bodies +$inceptionResult->getFiles(); +$container = $inceptionResult->getContainer(); +$analyser = $container->getByType(Analyser::class); +$finalizer = $container->getByType(AnalyserResultFinalizer::class); + +$analyse = static function (string $file) use ($analyser, $finalizer): int { + $result = $analyser->analyse([$file], null, null, true, [$file]); + + return count($finalizer->finalize($result, false, true)->getErrors()); +}; + +// A different file than the benchmark: PHPStan memoizes types and reflection in memory, +// so analysing the benchmark twice would hide work from the measured run +$analyse($warmupFile); + +$instance = null; +if ($hooks !== null) { + $instance = $hooks->instrument_hooks_init(); + if ($instance === null || !$hooks->instrument_hooks_is_instrumented($instance)) { + fwrite(STDERR, "Not running under CodSpeed's CPU simulation.\n"); + exit(1); + } + $hooks->instrument_hooks_set_integration($instance, 'phpstan-bench', '1.0.0'); +} + +memory_reset_peak_usage(); +if ($hooks !== null && $hooks->instrument_hooks_start_benchmark($instance) !== 0) { + fwrite(STDERR, "Cannot start the measurement.\n"); + exit(1); +} + +$errors = $analyse($benchmarkFile); + +if ($hooks !== null) { + if ($hooks->instrument_hooks_stop_benchmark($instance) !== 0) { + fwrite(STDERR, "Cannot stop the measurement.\n"); + exit(1); + } + // run.php finds the measured part of the profile by this name + $hooks->instrument_hooks_set_executed_benchmark($instance, getmypid(), 'tests/bench/data/' . basename($benchmarkFile)); +} + +file_put_contents($peakMemoryFile, (string) memory_get_peak_usage()); +fwrite(STDERR, sprintf("Analysed %s, %d errors.\n", $benchmarkFile, $errors)); diff --git a/tests/bench/phpstan.neon b/tests/bench/phpstan.neon new file mode 100644 index 00000000000..b79147e6d30 --- /dev/null +++ b/tests/bench/phpstan.neon @@ -0,0 +1,9 @@ +# Configuration the benchmarks in data/ are analysed with by run.php. +# data/ is excluded from the analysis of PHPStan itself. +includes: + - ../../conf/bleedingEdge.neon + +parameters: + level: 8 + inferPrivatePropertyTypeFromConstructor: true + tmpDir: ../../tmp/bench diff --git a/tests/bench/run.php b/tests/bench/run.php new file mode 100644 index 00000000000..084fcfe212c --- /dev/null +++ b/tests/bench/run.php @@ -0,0 +1,153 @@ + + */ + +use Nette\Neon\Neon; +use Nette\Utils\FileSystem; +use PHPStan\Process\Restart\OpcacheSettings; +use Symfony\Component\Process\Exception\ProcessTimedOutException; +use Symfony\Component\Process\Process; +use function basename; +use function count; +use function dirname; +use function file_get_contents; +use function file_put_contents; +use function fwrite; +use function getenv; +use function getmypid; +use function glob; +use function is_dir; +use function is_file; +use function mkdir; +use function realpath; +use function rmdir; +use function sprintf; +use function str_ends_with; +use function sys_get_temp_dir; +use function unlink; +use const PHP_BINARY; +use const STDERR; + +require_once __DIR__ . '/../../vendor/autoload.php'; + +// The slowest benchmark takes a few seconds when PHPStan runs natively, so a benchmark +// stuck in an infinite loop fails in the native run, before the much slower simulation +const NATIVE_RUN_SECONDS_LIMIT = 30; + +// The slowest benchmark takes about 13 minutes under CodSpeed's CPU simulation. A benchmark +// that regresses far beyond that fails as not measured instead of holding up its shard. +const SIMULATION_SECONDS_LIMIT = 25 * 60; + +if (!isset($argv) || count($argv) !== 3) { + fwrite(STDERR, "Usage: php tests/bench/run.php \n"); + exit(2); +} + +[, $benchmarkFile, $outputFile] = $argv; +$benchmark = basename($benchmarkFile); +$benchmarkFile = realpath($benchmarkFile); +if ($benchmarkFile === false) { + fwrite(STDERR, sprintf("Benchmark %s does not exist.\n", $argv[1])); + exit(2); +} + +// Each benchmark gets its own tmpDir, so what earlier benchmarks left in the cache does not change its numbers +$tmpDir = dirname(__DIR__, 2) . '/tmp/bench/' . $benchmark; +FileSystem::delete($tmpDir); +FileSystem::createDir($tmpDir); +$configFile = $tmpDir . '/phpstan.neon'; +FileSystem::write($configFile, Neon::encode([ + 'includes' => [__DIR__ . '/phpstan.neon'], + 'parameters' => ['tmpDir' => $tmpDir . '/cache'], +], true)); + +// measure.php does not restart PHP like bin/phpstan, so it gets the OPcache settings the restart +// would add. Valgrind also breaks PHP's detection of the stack size, so the stack limit check is +// turned off. +$command = [PHP_BINARY, '-d', 'ffi.enable=1', '-d', 'memory_limit=-1', '-d', 'zend.max_allowed_stack_size=-1']; +foreach (OpcacheSettings::getIniEntries() as $iniEntry) { + $command[] = '-d'; + $command[] = $iniEntry; +} +$peakMemoryFile = $tmpDir . '/peak-memory'; +$command = [...$command, 'tests/bench/measure.php', $configFile, $benchmarkFile, $peakMemoryFile]; + +$hooksLibrary = getenv('INSTRUMENT_HOOKS_LIBRARY'); +if ($hooksLibrary === false || !is_file($hooksLibrary)) { + fwrite(STDERR, "Set INSTRUMENT_HOOKS_LIBRARY to the path of CodSpeed's instrument-hooks library.\n"); + exit(2); +} + +// Compiles the DI container and fills the caches in tmpDir, so the measured run reads them +$warmUp = new Process($command, __DIR__ . '/../..', ['INSTRUMENT_HOOKS_LIBRARY' => false]); +$warmUp->setTimeout(NATIVE_RUN_SECONDS_LIMIT); +try { + $warmUp->run(); +} catch (ProcessTimedOutException) { + fwrite(STDERR, sprintf("PHPStan did not finish in %d seconds.\n", NATIVE_RUN_SECONDS_LIMIT)); + exit(1); +} +if (!$warmUp->isSuccessful()) { + fwrite(STDERR, $warmUp->getOutput() . $warmUp->getErrorOutput()); + fwrite(STDERR, sprintf("PHPStan failed with exit code %s.\n", $warmUp->getExitCode())); + exit(1); +} + +$profileDir = sys_get_temp_dir() . '/phpstan-benchmark-' . getmypid(); +if (!is_dir($profileDir) && !mkdir($profileDir, 0777, true)) { + fwrite(STDERR, sprintf("Cannot create %s.\n", $profileDir)); + exit(1); +} + +$codspeed = new Process([ + 'codspeed', + 'run', + '--mode=simulation', + // The comparison against expected.neon happens in compare.php + '--skip-upload', + '--profile-folder=' . $profileDir, + '--', + ...$command, +], __DIR__ . '/../..', ['INSTRUMENT_HOOKS_LIBRARY' => $hooksLibrary], timeout: SIMULATION_SECONDS_LIMIT); +try { + $codspeed->run(); +} catch (ProcessTimedOutException) { + fwrite(STDERR, sprintf("CodSpeed did not finish in %d minutes.\n", SIMULATION_SECONDS_LIMIT / 60)); + exit(1); +} + +// Every process gets its own file, only the one of measure.php has the measured part +$metrics = []; +foreach (glob($profileDir . '/*') ?: [] as $profileFile) { + if (str_ends_with($profileFile, '.out')) { + $profile = BenchmarkProfileParser::parseCallgrindOutput((string) file_get_contents($profileFile), 'tests/bench/data/' . $benchmark); + if ($profile !== null) { + $metrics[] = $profile; + } + } + unlink($profileFile); +} +rmdir($profileDir); + +$peakMemory = @file_get_contents($peakMemoryFile); +if (!$codspeed->isSuccessful() || count($metrics) !== 1 || $peakMemory === false) { + fwrite(STDERR, $codspeed->getOutput() . $codspeed->getErrorOutput()); + fwrite(STDERR, sprintf("CodSpeed exited with code %s and measured %d benchmarks instead of one.\n", $codspeed->getExitCode(), count($metrics))); + exit(1); +} + +$result = [$benchmark => $metrics[0] + ['peakMemory' => (int) $peakMemory]]; +if (file_put_contents($outputFile, Neon::encode($result, true)) === false) { + fwrite(STDERR, sprintf("Cannot write %s.\n", $outputFile)); + exit(1); +} + +echo Neon::encode($result, true); diff --git a/tests/bench/shard.php b/tests/bench/shard.php new file mode 100644 index 00000000000..50cef127df0 --- /dev/null +++ b/tests/bench/shard.php @@ -0,0 +1,45 @@ + + */ + +use Nette\Neon\Neon; +use function basename; +use function count; +use function file_get_contents; +use function fwrite; +use function glob; +use function is_file; +use function max; +use const STDERR; + +require_once __DIR__ . '/../../vendor/autoload.php'; + +if (!isset($argv) || count($argv) !== 3 || (int) $argv[1] < 1 || (int) $argv[2] < 1 || (int) $argv[2] > (int) $argv[1]) { + fwrite(STDERR, "Usage: php tests/bench/shard.php \n"); + exit(2); +} + +$expectedFile = __DIR__ . '/expected.neon'; +$expected = is_file($expectedFile) ? Neon::decode((string) file_get_contents($expectedFile)) : []; +$expectedInstructions = []; +foreach ($expected as $benchmark => $metrics) { + $expectedInstructions[$benchmark] = $metrics['instructions']; +} + +$costs = []; +foreach (glob(__DIR__ . '/data/*.php') ?: [] as $file) { + $benchmark = basename($file); + // a benchmark without expected metrics yet is usually a reproducer of something slow + $costs[$benchmark] = $expectedInstructions[$benchmark] ?? ($expectedInstructions === [] ? 1 : max($expectedInstructions)); +} + +foreach (BenchmarkShards::split($costs, (int) $argv[1])[(int) $argv[2] - 1] ?? [] as $benchmark) { + echo $benchmark . "\n"; +} diff --git a/tests/bench/storage/baseline.xml b/tests/bench/storage/baseline.xml deleted file mode 100644 index 92b51fc49af..00000000000 --- a/tests/bench/storage/baseline.xml +++ /dev/null @@ -1,965 +0,0 @@ - - - - - - Linux - runnervm8df0l - 6.17.0-1022-azure - #22-Ubuntu SMP Mon Jul 27 17:24:03 UTC 2026 - x86_64 - - - - 8.5.11 - /etc/php/8.5/cli/php.ini - Core, date, lexbor, openssl, pcre, zlib, filter, hash, json, uri, Zend OPcache, pcntl, random, Reflection, SPL, session, standard, sodium, libxml, mysqlnd, PDO, xml, amqp, apcu, ast, bcmath, bz2, calendar, ctype, curl, dba, dom, enchant, mbstring, FFI, fileinfo, ftp, gd, gettext, gmp, iconv, igbinary, imagick, imap, intl, ldap, exif, memcache, mongodb, msgpack, mysqli, odbc, pdo_dblib, PDO_Firebird, pdo_mysql, PDO_ODBC, pdo_pgsql, pdo_sqlite, pdo_sqlsrv, pgsql, Phar, posix, readline, shmop, SimpleXML, snmp, soap, sockets, sqlite3, sqlsrv, sysvmsg, sysvsem, sysvshm, tidy, tokenizer, xmlreader, xmlwriter, xsl, yaml, zip, zmq, memcached, redis, ds - - - 1 - - - - 0.32958984375 - 0.10546875 - 0.02978515625 - - - git - (unnamed branch) - - - - 0.0071525573730469 - 0.16903877258301 - 1.9309520721436 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/composer.json b/tests/composer.json index 38b351805e6..aaad8c1afd1 100644 --- a/tests/composer.json +++ b/tests/composer.json @@ -2,8 +2,7 @@ "name": "phpstan/phpstan-src-tests", "require-dev": { "phpunit/phpunit": "^11.5.23", - "brianium/paratest": "^7.8.3", - "phpbench/phpbench": "^1.7" + "brianium/paratest": "^7.8.3" }, "config": { "platform": { diff --git a/tests/composer.lock b/tests/composer.lock index 74dd996c1e6..299c1fd4196 100644 --- a/tests/composer.lock +++ b/tests/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e73ae0e785fcdc859478c2c8c1b32451", + "content-hash": "868be8e811ed8ef2531495ab93a0d835", "packages": [], "packages-dev": [ { @@ -100,158 +100,6 @@ ], "time": "2026-10-04T06:17:44+00:00" }, - { - "name": "doctrine/annotations", - "version": "2.0.2", - "source": { - "type": "git", - "url": "https://github.com/doctrine/annotations.git", - "reference": "901c2ee5d26eb64ff43c47976e114bf00843acf7" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/doctrine/annotations/zipball/901c2ee5d26eb64ff43c47976e114bf00843acf7", - "reference": "901c2ee5d26eb64ff43c47976e114bf00843acf7", - "shasum": "" - }, - "require": { - "doctrine/lexer": "^2 || ^3", - "ext-tokenizer": "*", - "php": "^7.2 || ^8.0", - "psr/cache": "^1 || ^2 || ^3" - }, - "require-dev": { - "doctrine/cache": "^2.0", - "doctrine/coding-standard": "^10", - "phpstan/phpstan": "^1.10.28", - "phpunit/phpunit": "^7.5 || ^8.5 || ^9.5", - "symfony/cache": "^5.4 || ^6.4 || ^7", - "vimeo/psalm": "^4.30 || ^5.14" - }, - "suggest": { - "php": "PHP 8.0 or higher comes with attributes, a native replacement for annotations" - }, - "type": "library", - "autoload": { - "psr-4": { - "Doctrine\\Common\\Annotations\\": "lib/Doctrine/Common/Annotations" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Guilherme Blanco", - "email": "guilhermeblanco@gmail.com" - }, - { - "name": "Roman Borschel", - "email": "roman@code-factory.org" - }, - { - "name": "Benjamin Eberlei", - "email": "kontakt@beberlei.de" - }, - { - "name": "Jonathan Wage", - "email": "jonwage@gmail.com" - }, - { - "name": "Johannes Schmitt", - "email": "schmittjoh@gmail.com" - } - ], - "description": "Docblock Annotations Parser", - "homepage": "https://www.doctrine-project.org/projects/annotations.html", - "keywords": [ - "annotations", - "docblock", - "parser" - ], - "support": { - "issues": "https://github.com/doctrine/annotations/issues", - "source": "https://github.com/doctrine/annotations/tree/2.0.2" - }, - "abandoned": true, - "time": "2024-09-05T10:17:24+00:00" - }, - { - "name": "doctrine/lexer", - "version": "3.0.3", - "source": { - "type": "git", - "url": "https://github.com/doctrine/lexer.git", - "reference": "71d305eaa6480d3d3460c7d053d1da7749fa9cca" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/doctrine/lexer/zipball/71d305eaa6480d3d3460c7d053d1da7749fa9cca", - "reference": "71d305eaa6480d3d3460c7d053d1da7749fa9cca", - "shasum": "" - }, - "require": { - "php": "^8.1" - }, - "require-dev": { - "doctrine/coding-standard": "^14", - "phpstan/phpstan": "^2", - "phpunit/phpunit": "^10.5.58 || ^12.5.4" - }, - "type": "library", - "autoload": { - "psr-4": { - "Doctrine\\Common\\Lexer\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Guilherme Blanco", - "email": "guilhermeblanco@gmail.com" - }, - { - "name": "Roman Borschel", - "email": "roman@code-factory.org" - }, - { - "name": "Johannes Schmitt", - "email": "schmittjoh@gmail.com" - } - ], - "description": "PHP Doctrine Lexer parser library that can be used in Top-Down, Recursive Descent Parsers.", - "homepage": "https://www.doctrine-project.org/projects/lexer.html", - "keywords": [ - "annotations", - "docblock", - "lexer", - "parser", - "php" - ], - "support": { - "issues": "https://github.com/doctrine/lexer/issues", - "source": "https://github.com/doctrine/lexer/tree/3.0.3" - }, - "funding": [ - { - "url": "https://www.doctrine-project.org/sponsorship.html", - "type": "custom" - }, - { - "url": "https://www.patreon.com/phpdoctrine", - "type": "patreon" - }, - { - "url": "https://tidelift.com/funding/github/packagist/doctrine%2Flexer", - "type": "tidelift" - } - ], - "time": "2026-10-01T06:59:29+00:00" - }, { "name": "fidry/cpu-core-counter", "version": "1.4.1", @@ -609,156 +457,6 @@ }, "time": "2022-02-21T01:04:05+00:00" }, - { - "name": "phpbench/container", - "version": "2.2.3", - "source": { - "type": "git", - "url": "https://github.com/phpbench/container.git", - "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpbench/container/zipball/0c7b2d36c1ea53fe27302fb8873ded7172047196", - "reference": "0c7b2d36c1ea53fe27302fb8873ded7172047196", - "shasum": "" - }, - "require": { - "psr/container": "^1.0|^2.0", - "symfony/options-resolver": "^4.2 || ^5.0 || ^6.0 || ^7.0 || ^8.0" - }, - "require-dev": { - "php-cs-fixer/shim": "^3.89", - "phpstan/phpstan": "^0.12.52", - "phpunit/phpunit": "^8" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.x-dev" - } - }, - "autoload": { - "psr-4": { - "PhpBench\\DependencyInjection\\": "lib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Daniel Leech", - "email": "daniel@dantleech.com" - } - ], - "description": "Simple, configurable, service container.", - "support": { - "issues": "https://github.com/phpbench/container/issues", - "source": "https://github.com/phpbench/container/tree/2.2.3" - }, - "time": "2025-11-06T09:05:13+00:00" - }, - { - "name": "phpbench/phpbench", - "version": "1.7.0", - "source": { - "type": "git", - "url": "https://github.com/phpbench/phpbench.git", - "reference": "3d13c0d5dcf8730a67b70fa7fb03b4556b5cc0fe" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/phpbench/phpbench/zipball/3d13c0d5dcf8730a67b70fa7fb03b4556b5cc0fe", - "reference": "3d13c0d5dcf8730a67b70fa7fb03b4556b5cc0fe", - "shasum": "" - }, - "require": { - "doctrine/annotations": "^2.0", - "ext-dom": "*", - "ext-json": "*", - "ext-pcre": "*", - "ext-reflection": "*", - "ext-spl": "*", - "ext-tokenizer": "*", - "php": "^8.2", - "phpbench/container": "^2.2", - "psr/log": "^1.1 || ^2.0 || ^3.0", - "seld/jsonlint": "^1.1", - "symfony/console": "^6.1 || ^7.0 || ^8.0", - "symfony/filesystem": "^6.1 || ^7.0 || ^8.0", - "symfony/finder": "^6.1 || ^7.0 || ^8.0", - "symfony/options-resolver": "^6.1 || ^7.0 || ^8.0", - "symfony/process": "^6.1 || ^7.0 || ^8.0", - "webmozart/glob": "^4.6" - }, - "require-dev": { - "dantleech/invoke": "^2.0", - "ergebnis/composer-normalize": "^2.39", - "jangregor/phpstan-prophecy": "^2.0", - "php-cs-fixer/shim": "^3.9", - "phpspec/prophecy": "^1.22", - "phpstan/extension-installer": "^1.1", - "phpstan/phpstan": "^2.0", - "phpstan/phpstan-phpunit": "^2.0", - "phpunit/phpunit": "^11.5", - "rector/rector": "^2.2", - "sebastian/exporter": "^6.3.2", - "symfony/error-handler": "^6.1 || ^7.0 || ^8.0", - "symfony/var-dumper": "^6.1 || ^7.0 || ^8.0" - }, - "suggest": { - "ext-xdebug": "For Xdebug profiling extension." - }, - "bin": [ - "bin/phpbench" - ], - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.2-dev" - } - }, - "autoload": { - "files": [ - "lib/Report/Func/functions.php" - ], - "psr-4": { - "PhpBench\\": "lib/", - "PhpBench\\Extensions\\XDebug\\": "extensions/xdebug/lib/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Daniel Leech", - "email": "daniel@dantleech.com" - } - ], - "description": "PHP Benchmarking Framework", - "keywords": [ - "benchmarking", - "optimization", - "performance", - "profiling", - "testing" - ], - "support": { - "issues": "https://github.com/phpbench/phpbench/issues", - "source": "https://github.com/phpbench/phpbench/tree/1.7.0" - }, - "funding": [ - { - "url": "https://github.com/dantleech", - "type": "github" - } - ], - "time": "2026-06-08T19:09:20+00:00" - }, { "name": "phpunit/php-code-coverage", "version": "11.0.12", @@ -1200,55 +898,6 @@ ], "time": "2026-10-05T05:47:54+00:00" }, - { - "name": "psr/cache", - "version": "3.0.0", - "source": { - "type": "git", - "url": "https://github.com/php-fig/cache.git", - "reference": "aa5030cfa5405eccfdcb1083ce040c2cb8d253bf" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-fig/cache/zipball/aa5030cfa5405eccfdcb1083ce040c2cb8d253bf", - "reference": "aa5030cfa5405eccfdcb1083ce040c2cb8d253bf", - "shasum": "" - }, - "require": { - "php": ">=8.0.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0.x-dev" - } - }, - "autoload": { - "psr-4": { - "Psr\\Cache\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "PHP-FIG", - "homepage": "https://www.php-fig.org/" - } - ], - "description": "Common interface for caching libraries", - "keywords": [ - "cache", - "psr", - "psr-6" - ], - "support": { - "source": "https://github.com/php-fig/cache/tree/3.0.0" - }, - "time": "2021-02-03T23:26:27+00:00" - }, { "name": "psr/container", "version": "2.0.2", @@ -1302,56 +951,6 @@ }, "time": "2021-11-05T16:47:00+00:00" }, - { - "name": "psr/log", - "version": "3.0.2", - "source": { - "type": "git", - "url": "https://github.com/php-fig/log.git", - "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", - "reference": "f16e1d5863e37f8d8c2a01719f5b34baa2b714d3", - "shasum": "" - }, - "require": { - "php": ">=8.0.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.x-dev" - } - }, - "autoload": { - "psr-4": { - "Psr\\Log\\": "src" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "PHP-FIG", - "homepage": "https://www.php-fig.org/" - } - ], - "description": "Common interface for logging libraries", - "homepage": "https://github.com/php-fig/log", - "keywords": [ - "log", - "psr", - "psr-3" - ], - "support": { - "source": "https://github.com/php-fig/log/tree/3.0.2" - }, - "time": "2024-09-11T13:17:53+00:00" - }, { "name": "sebastian/cli-parser", "version": "3.0.2", @@ -2338,70 +1937,6 @@ ], "time": "2024-10-09T05:16:32+00:00" }, - { - "name": "seld/jsonlint", - "version": "1.12.1", - "source": { - "type": "git", - "url": "https://github.com/Seldaek/jsonlint.git", - "reference": "9a90eb5d32d5a500296bf43f946d60246444d5f7" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/Seldaek/jsonlint/zipball/9a90eb5d32d5a500296bf43f946d60246444d5f7", - "reference": "9a90eb5d32d5a500296bf43f946d60246444d5f7", - "shasum": "" - }, - "require": { - "php": "^5.3 || ^7.0 || ^8.0" - }, - "require-dev": { - "phpstan/phpstan": "^1.11", - "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.0 || ^8.5.13" - }, - "bin": [ - "bin/jsonlint" - ], - "type": "library", - "autoload": { - "psr-4": { - "Seld\\JsonLint\\": "src/Seld/JsonLint/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Jordi Boggiano", - "email": "j.boggiano@seld.be", - "homepage": "https://seld.be" - } - ], - "description": "JSON Linter", - "keywords": [ - "json", - "linter", - "parser", - "validator" - ], - "support": { - "issues": "https://github.com/Seldaek/jsonlint/issues", - "source": "https://github.com/Seldaek/jsonlint/tree/1.12.1" - }, - "funding": [ - { - "url": "https://github.com/Seldaek", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/seld/jsonlint", - "type": "tidelift" - } - ], - "time": "2026-06-12T11:32:29+00:00" - }, { "name": "staabm/side-effects-detector", "version": "1.0.5", @@ -2623,215 +2158,6 @@ ], "time": "2026-06-05T06:23:12+00:00" }, - { - "name": "symfony/filesystem", - "version": "v7.4.21", - "source": { - "type": "git", - "url": "https://github.com/symfony/filesystem.git", - "reference": "fd0e3462be6d32a0d130e8423e4940a7688162d8" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/fd0e3462be6d32a0d130e8423e4940a7688162d8", - "reference": "fd0e3462be6d32a0d130e8423e4940a7688162d8", - "shasum": "" - }, - "require": { - "php": ">=8.2", - "symfony/polyfill-ctype": "~1.8", - "symfony/polyfill-mbstring": "~1.8" - }, - "require-dev": { - "symfony/process": "^6.4|^7.0|^8.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "Symfony\\Component\\Filesystem\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Provides basic utilities for the filesystem", - "homepage": "https://symfony.com", - "support": { - "source": "https://github.com/symfony/filesystem/tree/v7.4.21" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://github.com/nicolas-grekas", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2026-10-06T20:05:19+00:00" - }, - { - "name": "symfony/finder", - "version": "v7.4.20", - "source": { - "type": "git", - "url": "https://github.com/symfony/finder.git", - "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", - "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", - "shasum": "" - }, - "require": { - "php": ">=8.2" - }, - "require-dev": { - "symfony/filesystem": "^6.4|^7.0|^8.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "Symfony\\Component\\Finder\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Finds files and directories via an intuitive fluent interface", - "homepage": "https://symfony.com", - "support": { - "source": "https://github.com/symfony/finder/tree/v7.4.20" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://github.com/nicolas-grekas", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2026-09-26T06:31:10+00:00" - }, - { - "name": "symfony/options-resolver", - "version": "v7.4.8", - "source": { - "type": "git", - "url": "https://github.com/symfony/options-resolver.git", - "reference": "2888fcdc4dc2fd5f7c7397be78631e8af12e02b4" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/options-resolver/zipball/2888fcdc4dc2fd5f7c7397be78631e8af12e02b4", - "reference": "2888fcdc4dc2fd5f7c7397be78631e8af12e02b4", - "shasum": "" - }, - "require": { - "php": ">=8.2", - "symfony/deprecation-contracts": "^2.5|^3" - }, - "type": "library", - "autoload": { - "psr-4": { - "Symfony\\Component\\OptionsResolver\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Provides an improved replacement for the array_replace PHP function", - "homepage": "https://symfony.com", - "keywords": [ - "config", - "configuration", - "options" - ], - "support": { - "source": "https://github.com/symfony/options-resolver/tree/v7.4.8" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://github.com/nicolas-grekas", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2026-03-24T13:12:05+00:00" - }, { "name": "symfony/polyfill-ctype", "version": "v1.37.0", @@ -3459,55 +2785,6 @@ } ], "time": "2025-11-17T20:03:58+00:00" - }, - { - "name": "webmozart/glob", - "version": "4.7.0", - "source": { - "type": "git", - "url": "https://github.com/webmozarts/glob.git", - "reference": "8a2842112d6916e61e0e15e316465b611f3abc17" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/webmozarts/glob/zipball/8a2842112d6916e61e0e15e316465b611f3abc17", - "reference": "8a2842112d6916e61e0e15e316465b611f3abc17", - "shasum": "" - }, - "require": { - "php": "^7.3 || ^8.0.0" - }, - "require-dev": { - "phpunit/phpunit": "^9.5", - "symfony/filesystem": "^5.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "4.1-dev" - } - }, - "autoload": { - "psr-4": { - "Webmozart\\Glob\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Bernhard Schussek", - "email": "bschussek@gmail.com" - } - ], - "description": "A PHP implementation of Ant's glob.", - "support": { - "issues": "https://github.com/webmozarts/glob/issues", - "source": "https://github.com/webmozarts/glob/tree/4.7.0" - }, - "time": "2024-03-07T20:33:40+00:00" } ], "aliases": [], @@ -3520,5 +2797,5 @@ "platform-overrides": { "php": "8.2.99" }, - "plugin-api-version": "2.9.0" + "plugin-api-version": "2.11.0" }