From 67f684d48758b23b9ad6a2e7beb71e2ecad952f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Fri, 6 Nov 2026 00:18:30 +0100 Subject: [PATCH 1/7] add "download-bundled" generator --- .../actions/verify-generated-files/action.yml | 1 + .../download-bundled/make-workflow-file.php | 214 ++++++++++++++++++ .github/workflows/verify-bundled-files.yml | 6 +- 3 files changed, 218 insertions(+), 3 deletions(-) create mode 100755 .github/scripts/download-bundled/make-workflow-file.php diff --git a/.github/actions/verify-generated-files/action.yml b/.github/actions/verify-generated-files/action.yml index 79c49dbfcfff..3f3dea73c4dc 100644 --- a/.github/actions/verify-generated-files/action.yml +++ b/.github/actions/verify-generated-files/action.yml @@ -13,4 +13,5 @@ runs: ext/tokenizer/tokenizer_data_gen.php build/gen_stub.php -f --generate-optimizer-info --verify ext/phar/makestub.php + .github/scripts/download-bundled/make-workflow-file.php .github/scripts/test-directory-unchanged.sh . diff --git a/.github/scripts/download-bundled/make-workflow-file.php b/.github/scripts/download-bundled/make-workflow-file.php new file mode 100755 index 000000000000..0268803a91bb --- /dev/null +++ b/.github/scripts/download-bundled/make-workflow-file.php @@ -0,0 +1,214 @@ +#!/usr/bin/env php + $directories + * @param list $files + */ + public function __construct( + public string $name, + public array $directories, + public array $files = [] + ) {} + + public function getNameForPath(): string + { + return preg_replace('~\W+~', '-', strtolower($this->name)); + } +} + +class Generator +{ + /** + * @param list $bundles + */ + public function __construct( + public array $bundles + ) {} + + protected function getRepoDirectory(): string + { + return dirname(__DIR__, 3); + } + + protected function indentString(string $value, int $levels, bool $inclFirstLine): string + { + return preg_replace( + '~' . ($inclFirstLine ? '^|' : '') . '(?<=\n)~', + str_repeat(' ', $levels), + $value + ); + } + + /** + * @param mixed $data + */ + protected function encodeYml($data): string + { + if (is_array($data)) { + $isList = array_is_list($data); + $resParts = []; + foreach ($data as $k => $v) { + $kEncoded = $isList + ? '-' + : $this->encodeYml($k) . ':'; + $vEncoded = $this->encodeYml($v); + + $resParts[] = $kEncoded + . (!$isList && is_array($v) && $v !== [] ? "\n " : ' ') + . (is_array($v) ? $this->indentString($vEncoded, 1, false) : $vEncoded); + } + + return implode("\n", $resParts); + } + + if (preg_match('~^(\w+|\$\{\{[^\}]+\}\})$~', $data)) { + return $data; + } + + return strpos($data, "\n") !== false + ? '|' . "\n" . $this->indentString($data, 1, true) + : '\'' . str_replace('\'', '\'\'', $data) . '\''; + } + + public function makeWorkflowFile(): void + { + $content = <<<'EOD' + name: Verify Bundled Files + + on: + push: + paths: &paths + %paths% + pull_request: + paths: *paths + schedule: + - cron: "0 1 * * *" + workflow_dispatch: ~ + + permissions: + contents: read + + jobs: + VERIFY_BUNDLED_FILES: + if: github.repository == 'php/php-src' || github.event_name == 'workflow_dispatch' + name: Verify Bundled Files + runs-on: ubuntu-24.04 + steps: + - name: git checkout + uses: actions/checkout@v7 + + - name: Detect changed files + if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} + uses: dorny/paths-filter@v3 + id: changes + with: + filters: %filters% + + %steps% + + EOD; + + $paths = [ + '.github/scripts/download-bundled/**', + ]; + foreach ($this->bundles as $bundle) { + foreach ($this->makeDornyPathsFilterFilters($bundle) as $p) { + if (str_starts_with($p, '.github/scripts/download-bundled/')) { + continue; + } + + $paths[] = $p; + } + } + $content = str_replace('%paths%', $this->indentString($this->encodeYml($paths), 3, false), $content); + + $filters = []; + foreach ($this->bundles as $bundle) { + $filters[$bundle->getNameForPath()] = $this->makeDornyPathsFilterFilters($bundle); + } + $content = str_replace('%filters%', $this->indentString($this->encodeYml($this->encodeYml($filters)), 5, false), $content); + + $steps = []; + foreach ($this->bundles as $bundle) { + $steps[] = [ + 'name' => $bundle->name, + 'if' => '${{ !cancelled() && (steps.changes.outputs.' . $bundle->getNameForPath() . ' == \'true\' || github.event_name == \'schedule\' || github.event_name == \'workflow_dispatch\') }}', + 'run' => implode("\n", [ + 'echo "::group::Download"', + '.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.sh', + 'echo "::endgroup::"', + 'echo "::group::Verify files"', + ...array_map(static fn ($v) => '.github/scripts/test-directory-unchanged.sh \'' . $v . '\'', array_merge($bundle->directories, $bundle->files)), + 'echo "::endgroup::"', + ]), + ]; + } + $content = str_replace('%steps%', $this->indentString($this->encodeYml($steps), 3, false), $content); + + file_put_contents($this->getRepoDirectory() . '/.github/workflows/verify-bundled-files.yml', $content); + } + + protected function makeDornyPathsFilterFilters(Bundle $bundle): array + { + return [ + '.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.*', + ...array_map(static fn ($v) => $v . '/**', $bundle->directories), + ...$bundle->files, + ]; + } + + public function makeDownloadScriptHeaders(): void + { + foreach ($this->bundles as $bundle) { + $this->makeDownloadScriptHeader($bundle); + } + } + + protected function makeDownloadScriptHeader(Bundle $bundle): void + { + $scriptPath = $this->getRepoDirectory() . '/.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.sh'; + + $content = !file_exists($scriptPath) + ? "# TODO\n" + : file_get_contents($scriptPath); + + $header = <<<'EOD' + #!/bin/sh + set -ex + cd "$(dirname "$0")/../../.." + + tmp_dir=%tmp_dir% + rm -rf "$tmp_dir" + + + EOD; + + $header = str_replace('%tmp_dir%', '/tmp/php-src-download-bundled/' . $bundle->getNameForPath(), $header); + + if (!str_starts_with($content, $header)) { + $content = $header . $content; + } + + file_put_contents($scriptPath, $content); + } +} + +$generator = new Generator($bundles); +$generator->makeWorkflowFile(); +$generator->makeDownloadScriptHeaders(); diff --git a/.github/workflows/verify-bundled-files.yml b/.github/workflows/verify-bundled-files.yml index a287e1229305..069ff97fe90c 100644 --- a/.github/workflows/verify-bundled-files.yml +++ b/.github/workflows/verify-bundled-files.yml @@ -60,7 +60,7 @@ jobs: .github/scripts/download-bundled/boost-context.sh echo "::endgroup::" echo "::group::Verify files" - .github/scripts/test-directory-unchanged.sh Zend/asm + .github/scripts/test-directory-unchanged.sh 'Zend/asm' echo "::endgroup::" - name: XSSE if: ${{ !cancelled() && (steps.changes.outputs.xsse == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} @@ -96,7 +96,7 @@ jobs: .github/scripts/download-bundled/pcre2.sh echo "::endgroup::" echo "::group::Verify files" - .github/scripts/test-directory-unchanged.sh ext/pcre/pcre2lib + .github/scripts/test-directory-unchanged.sh 'ext/pcre/pcre2lib' echo "::endgroup::" - name: uriparser if: ${{ !cancelled() && (steps.changes.outputs.uriparser == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} @@ -105,5 +105,5 @@ jobs: .github/scripts/download-bundled/uriparser.sh echo "::endgroup::" echo "::group::Verify files" - .github/scripts/test-directory-unchanged.sh ext/uri/uriparser + .github/scripts/test-directory-unchanged.sh 'ext/uri/uriparser' echo "::endgroup::" From cda670f110766edde599f23b3693b0f4a90b0e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Fri, 6 Nov 2026 00:46:39 +0100 Subject: [PATCH 2/7] add manual --- .../download-bundled/make-workflow-file.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/scripts/download-bundled/make-workflow-file.php b/.github/scripts/download-bundled/make-workflow-file.php index 0268803a91bb..0892f8fb0fc3 100755 --- a/.github/scripts/download-bundled/make-workflow-file.php +++ b/.github/scripts/download-bundled/make-workflow-file.php @@ -5,6 +5,24 @@ namespace Phpsrc\Ci\DownloadBundled; +// How to add a new bundled source? +// 1. add a new entry for the source into $bundles variable below +// 2. run this make-workflow-file.php generator +// 3. edit the generated .github/scripts/download-bundled/.sh file, see and be consistent with other files + +// How to run this make-workflow-file.php generator? +// ``` +// docker run --rm -v "$(git rev-parse --show-toplevel)":/h php:latest /h/.github/scripts/download-bundled/make-workflow-file.php +// ``` + +// How to update/download specific bundled source? +// 1. edit the bundled version inside relevant .github/scripts/download-bundled/.sh file +// 2. run: +// ``` +// docker run --rm -v "$(git rev-parse --show-toplevel)":/f alpine:latest sh -c "apk add git && /f/.github/scripts/download-bundled/.sh" +// ``` +// 3. review git changes and commit the changes + $bundles = [ new Bundle('boost.context', ['Zend/asm']), new Bundle('XSSE', [], ['Zend/zend_simd.h']), From 0022a7a658ad9649675605c3ca4b2a7ca1ac634b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Fri, 6 Nov 2026 18:21:22 +0100 Subject: [PATCH 3/7] address feedback - YAML nowdoc --- .github/scripts/download-bundled/make-workflow-file.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/download-bundled/make-workflow-file.php b/.github/scripts/download-bundled/make-workflow-file.php index 0892f8fb0fc3..eb7e70ed4005 100755 --- a/.github/scripts/download-bundled/make-workflow-file.php +++ b/.github/scripts/download-bundled/make-workflow-file.php @@ -106,7 +106,7 @@ protected function encodeYml($data): string public function makeWorkflowFile(): void { - $content = <<<'EOD' + $content = <<<'YAML' name: Verify Bundled Files on: @@ -140,7 +140,7 @@ public function makeWorkflowFile(): void %steps% - EOD; + YAML; $paths = [ '.github/scripts/download-bundled/**', @@ -206,7 +206,7 @@ protected function makeDownloadScriptHeader(Bundle $bundle): void ? "# TODO\n" : file_get_contents($scriptPath); - $header = <<<'EOD' + $header = <<<'YAML' #!/bin/sh set -ex cd "$(dirname "$0")/../../.." @@ -215,7 +215,7 @@ protected function makeDownloadScriptHeader(Bundle $bundle): void rm -rf "$tmp_dir" - EOD; + YAML; $header = str_replace('%tmp_dir%', '/tmp/php-src-download-bundled/' . $bundle->getNameForPath(), $header); From c456688edb6ebbf269b696db4d4d029feca5a3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Fri, 6 Nov 2026 18:23:04 +0100 Subject: [PATCH 4/7] address feedback - better method name for "dorny filters" --- .github/scripts/download-bundled/make-workflow-file.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/download-bundled/make-workflow-file.php b/.github/scripts/download-bundled/make-workflow-file.php index eb7e70ed4005..703c6ef46cf0 100755 --- a/.github/scripts/download-bundled/make-workflow-file.php +++ b/.github/scripts/download-bundled/make-workflow-file.php @@ -146,7 +146,7 @@ public function makeWorkflowFile(): void '.github/scripts/download-bundled/**', ]; foreach ($this->bundles as $bundle) { - foreach ($this->makeDornyPathsFilterFilters($bundle) as $p) { + foreach ($this->makeFiltersForDornyPathsFilterAction($bundle) as $p) { if (str_starts_with($p, '.github/scripts/download-bundled/')) { continue; } @@ -158,7 +158,7 @@ public function makeWorkflowFile(): void $filters = []; foreach ($this->bundles as $bundle) { - $filters[$bundle->getNameForPath()] = $this->makeDornyPathsFilterFilters($bundle); + $filters[$bundle->getNameForPath()] = $this->makeFiltersForDornyPathsFilterAction($bundle); } $content = str_replace('%filters%', $this->indentString($this->encodeYml($this->encodeYml($filters)), 5, false), $content); @@ -182,7 +182,7 @@ public function makeWorkflowFile(): void file_put_contents($this->getRepoDirectory() . '/.github/workflows/verify-bundled-files.yml', $content); } - protected function makeDornyPathsFilterFilters(Bundle $bundle): array + protected function makeFiltersForDornyPathsFilterAction(Bundle $bundle): array { return [ '.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.*', From 4ddc25e5f1c3eab9a709ed1d4af203767628f298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Fri, 6 Nov 2026 18:27:27 +0100 Subject: [PATCH 5/7] address feedback - describe what generates the workflow file --- .github/scripts/download-bundled/make-workflow-file.php | 3 +++ .github/workflows/verify-bundled-files.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/scripts/download-bundled/make-workflow-file.php b/.github/scripts/download-bundled/make-workflow-file.php index 703c6ef46cf0..9a0b7dd693cc 100755 --- a/.github/scripts/download-bundled/make-workflow-file.php +++ b/.github/scripts/download-bundled/make-workflow-file.php @@ -107,6 +107,9 @@ protected function encodeYml($data): string public function makeWorkflowFile(): void { $content = <<<'YAML' + # + # whole file is generated by .github/scripts/download-bundled/make-workflow-file.php + # name: Verify Bundled Files on: diff --git a/.github/workflows/verify-bundled-files.yml b/.github/workflows/verify-bundled-files.yml index 069ff97fe90c..35b08eeeb74a 100644 --- a/.github/workflows/verify-bundled-files.yml +++ b/.github/workflows/verify-bundled-files.yml @@ -1,3 +1,6 @@ +# +# whole file is generated by .github/scripts/download-bundled/make-workflow-file.php +# name: Verify Bundled Files on: From 298995c92204e367945d6542f32d016f83328e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Fri, 6 Nov 2026 19:04:53 +0100 Subject: [PATCH 6/7] address feedback - allow to update header and describy what generates it --- .github/scripts/download-bundled/boost-context.sh | 1 + .github/scripts/download-bundled/jit-ir.sh | 1 + .../scripts/download-bundled/make-workflow-file.php | 11 ++++------- .github/scripts/download-bundled/pcre2.sh | 1 + .github/scripts/download-bundled/uriparser.sh | 1 + .github/scripts/download-bundled/xsse.sh | 1 + .github/scripts/download-bundled/xxhash.sh | 1 + 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/scripts/download-bundled/boost-context.sh b/.github/scripts/download-bundled/boost-context.sh index 02febff73c3f..5501a3ce2131 100755 --- a/.github/scripts/download-bundled/boost-context.sh +++ b/.github/scripts/download-bundled/boost-context.sh @@ -4,6 +4,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/boost-context rm -rf "$tmp_dir" +# header above this line is generated by make-workflow-file.php revision=refs/tags/boost-1.91.0 diff --git a/.github/scripts/download-bundled/jit-ir.sh b/.github/scripts/download-bundled/jit-ir.sh index 9a1f8faa1be1..fc6cfe2c6843 100755 --- a/.github/scripts/download-bundled/jit-ir.sh +++ b/.github/scripts/download-bundled/jit-ir.sh @@ -4,6 +4,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/jit-ir rm -rf "$tmp_dir" +# header above this line is generated by make-workflow-file.php revision=4a1208ccbfc8ca32a5103baf9820ad3cbb6ff06e diff --git a/.github/scripts/download-bundled/make-workflow-file.php b/.github/scripts/download-bundled/make-workflow-file.php index 9a0b7dd693cc..20d02b977576 100755 --- a/.github/scripts/download-bundled/make-workflow-file.php +++ b/.github/scripts/download-bundled/make-workflow-file.php @@ -205,9 +205,9 @@ protected function makeDownloadScriptHeader(Bundle $bundle): void { $scriptPath = $this->getRepoDirectory() . '/.github/scripts/download-bundled/' . $bundle->getNameForPath() . '.sh'; - $content = !file_exists($scriptPath) + $contentWithoutHeader = !file_exists($scriptPath) ? "# TODO\n" - : file_get_contents($scriptPath); + : preg_replace('~^.*\n# header above this line is generated by make-workflow-file.php\n+~s', '', file_get_contents($scriptPath)); $header = <<<'YAML' #!/bin/sh @@ -216,17 +216,14 @@ protected function makeDownloadScriptHeader(Bundle $bundle): void tmp_dir=%tmp_dir% rm -rf "$tmp_dir" + # header above this line is generated by make-workflow-file.php YAML; $header = str_replace('%tmp_dir%', '/tmp/php-src-download-bundled/' . $bundle->getNameForPath(), $header); - if (!str_starts_with($content, $header)) { - $content = $header . $content; - } - - file_put_contents($scriptPath, $content); + file_put_contents($scriptPath, $header . $contentWithoutHeader); } } diff --git a/.github/scripts/download-bundled/pcre2.sh b/.github/scripts/download-bundled/pcre2.sh index 360cbb91db82..0452f1fbc238 100755 --- a/.github/scripts/download-bundled/pcre2.sh +++ b/.github/scripts/download-bundled/pcre2.sh @@ -4,6 +4,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/pcre2 rm -rf "$tmp_dir" +# header above this line is generated by make-workflow-file.php revision=refs/tags/pcre2-10.44 diff --git a/.github/scripts/download-bundled/uriparser.sh b/.github/scripts/download-bundled/uriparser.sh index 34d4b7fc5de5..441b7c718b5a 100755 --- a/.github/scripts/download-bundled/uriparser.sh +++ b/.github/scripts/download-bundled/uriparser.sh @@ -4,6 +4,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/uriparser rm -rf "$tmp_dir" +# header above this line is generated by make-workflow-file.php revision=refs/tags/uriparser-1.0.3 diff --git a/.github/scripts/download-bundled/xsse.sh b/.github/scripts/download-bundled/xsse.sh index 1e2c94d9366f..22965f20e9f3 100755 --- a/.github/scripts/download-bundled/xsse.sh +++ b/.github/scripts/download-bundled/xsse.sh @@ -4,6 +4,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/xsse rm -rf "$tmp_dir" +# header above this line is generated by make-workflow-file.php revision=refs/tags/xsse-1.0.0 diff --git a/.github/scripts/download-bundled/xxhash.sh b/.github/scripts/download-bundled/xxhash.sh index fc0c7eb7b302..0d5d8bab46f1 100755 --- a/.github/scripts/download-bundled/xxhash.sh +++ b/.github/scripts/download-bundled/xxhash.sh @@ -4,6 +4,7 @@ cd "$(dirname "$0")/../../.." tmp_dir=/tmp/php-src-download-bundled/xxhash rm -rf "$tmp_dir" +# header above this line is generated by make-workflow-file.php revision=refs/tags/v0.8.2 From 8177d262c304fbd966d69462fb19a48d06c7eaaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Vo=C5=99=C3=AD=C5=A1ek?= Date: Fri, 6 Nov 2026 19:10:40 +0100 Subject: [PATCH 7/7] address feedback - add comment about path being generated --- .github/scripts/download-bundled/make-workflow-file.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/download-bundled/make-workflow-file.php b/.github/scripts/download-bundled/make-workflow-file.php index 20d02b977576..69ccc189b0db 100755 --- a/.github/scripts/download-bundled/make-workflow-file.php +++ b/.github/scripts/download-bundled/make-workflow-file.php @@ -6,7 +6,7 @@ namespace Phpsrc\Ci\DownloadBundled; // How to add a new bundled source? -// 1. add a new entry for the source into $bundles variable below +// 1. add a new entry for the source into $bundles variable below (name and monitored directories/files, path is generated from the name) // 2. run this make-workflow-file.php generator // 3. edit the generated .github/scripts/download-bundled/.sh file, see and be consistent with other files