diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index b9df7364e26..952cb69cf17 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -124,6 +124,11 @@ jobs: mv src/Foo.php.orig src/Foo.php echo -n > phpstan-baseline.neon ../../bin/phpstan -vvv + - script: | + cd e2e/result-cache-truncated + ../../bin/phpstan -vvv + php truncate.php + ../../bin/phpstan -vvv - script: | cd e2e/bug-14514 composer install diff --git a/.gitignore b/.gitignore index a19ae4998f6..a2cdd4a7433 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ tmp/.memory_limit e2e/bashunit /.phpbench +/e2e/result-cache-truncated/tmp diff --git a/e2e/result-cache-truncated/phpstan.neon b/e2e/result-cache-truncated/phpstan.neon new file mode 100644 index 00000000000..6daefc8c39c --- /dev/null +++ b/e2e/result-cache-truncated/phpstan.neon @@ -0,0 +1,5 @@ +parameters: + level: 8 + tmpDir: tmp + paths: + - src diff --git a/e2e/result-cache-truncated/src/Bar.php b/e2e/result-cache-truncated/src/Bar.php new file mode 100644 index 00000000000..8b38f8ea2a6 --- /dev/null +++ b/e2e/result-cache-truncated/src/Bar.php @@ -0,0 +1,13 @@ +doBar(); + } + +} diff --git a/e2e/result-cache-truncated/truncate.php b/e2e/result-cache-truncated/truncate.php new file mode 100644 index 00000000000..ad3b7e98508 --- /dev/null +++ b/e2e/result-cache-truncated/truncate.php @@ -0,0 +1,13 @@ + is never reached), so a downgrade + * degrades to a silent full analysis instead. + */ + private const SERIALIZED_FILE_PREFIX = ''; + /** @var array */ private array $fileHashes = []; @@ -211,7 +227,12 @@ public function restore(array $allAnalysedFiles, bool $debug, bool $onlyFiles, ? } try { - $data = require $cacheFilePath; + // The cache used to be a var_export'd PHP file loaded via include. Including a + // multi-megabyte PHP source retains its compiled op_arrays and interned strings + // for the process lifetime; unserialize() produces only the values. A cache file + // in the old PHP format fails to unserialize and is discarded below like any + // other corrupted file, so no cache version bump is needed for the transition. + $data = $this->readCacheFile($cacheFilePath); } catch (Throwable $e) { if ($output->isVeryVerbose()) { $output->writeLineFormatted(sprintf('Result cache not used because an error occurred while loading the cache file: %s', $e->getMessage())); @@ -449,12 +470,12 @@ public function restore(array $allAnalysedFiles, bool $debug, bool $onlyFiles, ? $filesToAnalyse = []; $invertedDependenciesToReturn = []; $invertedUsedTraitDependenciesToReturn = []; - $errors = $data['errorsCallback'](); - $locallyIgnoredErrors = $data['locallyIgnoredErrorsCallback'](); + $errors = $data['errors']; + $locallyIgnoredErrors = $data['locallyIgnoredErrors']; $linesToIgnore = $data['linesToIgnore']; $unmatchedLineIgnores = $data['unmatchedLineIgnores']; - $collectedData = $data['collectedDataCallback'](); - $exportedNodes = $data['exportedNodesCallback'](); + $collectedData = $data['collectedData']; + $exportedNodes = $data['exportedNodes']; $filteredErrors = []; $filteredLocallyIgnoredErrors = []; $filteredLinesToIgnore = []; @@ -1212,9 +1233,11 @@ private function save( $file = $this->cacheFilePath; - // streamed to the file section by section - building the whole - // var_export()ed contents in memory at once would take up roughly - // twice the size of the resulting file in the main process + // Written frame by frame, and the array sections entry by entry, so the peak cost of saving is + // one entry rather than the whole cache. Serializing the payload in one call would hold the + // entire cache in memory twice over - on one project that is 53 MB serialized, 39 MB of it + // exportedNodes alone - which is the same trap the var_export writer this replaces avoided by + // streaming. $handle = @fopen($file, 'w'); if ($handle === false) { $error = error_get_last(); @@ -1222,38 +1245,18 @@ private function save( } try { - $this->writeToHandle($handle, $file, " " . var_export($lastFullAnalysisTime, true) . ", - 'meta' => " . var_export($meta, true) . ", - 'projectExtensionFiles' => " . var_export($projectExtensionFiles, true) . ", - 'errorsCallback' => static function (): array { return "); - $this->streamArrayVarExportToHandle($handle, $file, $errors); - $this->writeToHandle($handle, $file, "; }, - 'locallyIgnoredErrorsCallback' => static function (): array { return "); - $this->streamArrayVarExportToHandle($handle, $file, $locallyIgnoredErrors); - $this->writeToHandle($handle, $file, "; }, - 'linesToIgnore' => "); - $this->streamArrayVarExportToHandle($handle, $file, $linesToIgnore); - $this->writeToHandle($handle, $file, ", - 'unmatchedLineIgnores' => "); - $this->streamArrayVarExportToHandle($handle, $file, $unmatchedLineIgnores); - $this->writeToHandle($handle, $file, ", - 'collectedDataCallback' => static function (): array { return "); - $this->streamArrayVarExportToHandle($handle, $file, $collectedData); - $this->writeToHandle($handle, $file, "; }, - 'dependencies' => "); - $this->streamArrayVarExportToHandle($handle, $file, $invertedDependencies); - $this->writeToHandle($handle, $file, ", - 'packageDependencies' => "); - $this->streamArrayVarExportToHandle($handle, $file, $packageDependencies); - $this->writeToHandle($handle, $file, ", - 'exportedNodesCallback' => static function (): array { return "); - $this->streamArrayVarExportToHandle($handle, $file, $exportedNodes); - $this->writeToHandle($handle, $file, '; }, -]; -'); + $this->writeToHandle($handle, $file, self::SERIALIZED_FILE_PREFIX . "\n"); + $this->writeValueFrame($handle, $file, 'lastFullAnalysisTime', $lastFullAnalysisTime); + $this->writeValueFrame($handle, $file, 'meta', $meta); + $this->writeValueFrame($handle, $file, 'projectExtensionFiles', $projectExtensionFiles); + $this->writeArrayFrame($handle, $file, 'errors', $errors); + $this->writeArrayFrame($handle, $file, 'locallyIgnoredErrors', $locallyIgnoredErrors); + $this->writeArrayFrame($handle, $file, 'linesToIgnore', $linesToIgnore); + $this->writeArrayFrame($handle, $file, 'unmatchedLineIgnores', $unmatchedLineIgnores); + $this->writeArrayFrame($handle, $file, 'collectedData', $collectedData); + $this->writeArrayFrame($handle, $file, 'dependencies', $invertedDependencies); + $this->writeArrayFrame($handle, $file, 'packageDependencies', $packageDependencies); + $this->writeArrayFrame($handle, $file, 'exportedNodes', $exportedNodes); } finally { fclose($handle); } @@ -1271,30 +1274,147 @@ private function writeToHandle($handle, string $file, string $contents): void } /** - * Streams the var_export() representation of an array to the file entry - * by entry, producing output byte-identical to var_export($values, true). + * A single value, as `name length\n` followed by that many bytes. * - * var_export() builds the whole export in memory even when told to print it, - * so exporting a big section in one call would take up as much memory - * as the resulting file section itself. + * @param resource $handle + */ + private function writeValueFrame($handle, string $file, string $name, mixed $value): void + { + $blob = serialize($value); + $this->writeToHandle($handle, $file, $name . ' ' . strlen($blob) . "\n"); + $this->writeToHandle($handle, $file, $blob); + } + + /** + * An array, as `name* count\n` followed by one length-prefixed frame per entry. * - * Each entry is exported wrapped in a single-entry array whose "array (\n" - * prefix and "\n)" suffix are stripped, yielding the same bytes (including - * indentation) the entry would get inside the full export. Indenting the lines - * of a standalone value export would corrupt multi-line string contents instead. + * Each entry is serialized as a single-element array so its key travels with it, which keeps string + * and integer keys distinct without a second frame for the key. * * @param resource $handle * @param array $values */ - private function streamArrayVarExportToHandle($handle, string $file, array $values): void + private function writeArrayFrame($handle, string $file, string $name, array $values): void { - $this->writeToHandle($handle, $file, 'array ('); + $this->writeToHandle($handle, $file, $name . '* ' . count($values) . "\n"); foreach ($values as $key => $value) { - $entry = var_export([$key => $value], true); - $this->writeToHandle($handle, $file, "\n" . substr($entry, 8, -2)); + $blob = serialize([$key => $value]); + $this->writeToHandle($handle, $file, strlen($blob) . "\n"); + $this->writeToHandle($handle, $file, $blob); + } + } + + /** + * Read a framed cache file back, one frame at a time. + * + * Returns null for anything that is not this format, which is how a cache written by an older + * PHPStan is detected: the caller discards it and analyses everything, exactly as it does for a + * corrupted file. + * + * @return array|null + */ + private function readCacheFile(string $cacheFilePath): ?array + { + $handle = @fopen($cacheFilePath, 'r'); + if ($handle === false) { + return null; + } + + try { + if (rtrim((string) fgets($handle), "\n") !== self::SERIALIZED_FILE_PREFIX) { + return null; + } + + $data = []; + while (($header = fgets($handle)) !== false) { + $header = rtrim($header, "\n"); + if ($header === '') { + continue; + } + + $parts = explode(' ', $header, 2); + if (count($parts) !== 2) { + throw new RuntimeException(sprintf('Malformed frame header "%s".', $header)); + } + + [$name, $size] = $parts; + if (!str_ends_with($name, '*')) { + $data[$name] = $this->readFrame($handle, (int) $size); + + continue; + } + + $data[substr($name, 0, -1)] = $this->readEntryFrames($handle, (int) $size); + } + + return $data; + } finally { + fclose($handle); + } + } + + /** + * @param resource $handle + * @return array + */ + private function readEntryFrames($handle, int $count): array + { + $entries = []; + for ($i = 0; $i < $count; $i++) { + $length = fgets($handle); + if ($length === false) { + throw new RuntimeException(sprintf('Cache file ended after %d of %d entries.', $i, $count)); + } + + $entry = $this->readFrame($handle, (int) rtrim($length, "\n")); + if (!is_array($entry)) { + throw new RuntimeException('An entry frame did not contain an array.'); + } + + foreach ($entry as $key => $value) { + $entries[$key] = $value; + } + } + + return $entries; + } + + /** + * A frame's payload, or an exception when the file does not hold one. + * + * Every failure here means a cache file that is this format but damaged - a process killed + * mid-save leaves exactly that, since the file is written in place. restore() turns the + * exception into a discarded cache and a full analysis, the same way it handles the parse + * error an incomplete var_export'd file used to produce. Returning a value instead would + * hand a half-read cache to the caller, where the missing pieces surface as type errors far + * from the cause. + * + * false is treated as failure because unserialize() reports failure that way and no value in + * the cache is a bare false: the sections are arrays and lastFullAnalysisTime is an int. + * + * @param resource $handle + */ + private function readFrame($handle, int $length): mixed + { + if ($length <= 0) { + throw new RuntimeException(sprintf('Frame length %d is not positive.', $length)); + } + + $blob = fread($handle, $length); + if ($blob === false || strlen($blob) !== $length) { + throw new RuntimeException(sprintf( + 'Expected a %d byte frame, read %d bytes.', + $length, + $blob === false ? 0 : strlen($blob), + )); + } + + $value = @unserialize($blob); + if ($value === false) { + throw new RuntimeException(sprintf('A %d byte frame could not be unserialized.', $length)); } - $this->writeToHandle($handle, $file, "\n)"); + return $value; } /**