diff --git a/skills/dload-fetch-tool/references/registry-entry.md b/skills/dload-fetch-tool/references/registry-entry.md index 005369e..b9a75ee 100644 --- a/skills/dload-fetch-tool/references/registry-entry.md +++ b/skills/dload-fetch-tool/references/registry-entry.md @@ -22,14 +22,14 @@ Filters the release's asset list. dload then runs OS/arch detection on every mat - match all OS/arch variants of the tool; - leave out sibling tools, checksums, signatures, source archives. -Build variants stay in the pattern too. Among the assets for the host, dload prefers the lowest x86-64 microarchitecture level (`amd64` or `amd64_v1` over `amd64_v2`, `_v3`, `_v4`, also after `x86_64` or `x86-64`: the CPU is not checked, so a higher level is only a fallback, still picked when it is the only build for the host), then the host libc (`musl` builds on Alpine and Android, the others elsewhere), then the name with the fewest extra tokens — so `bun-linux-x64.zip` wins over `-baseline`, `-profile` and `-debug` twins — then archives. Android builds are dropped on Linux whenever the entry has a `binary` (without one, every other platform is only ranked lower), and so are OS packages (`.deb`, `.rpm`, `.apk`, `.snap`, `.flatpak`, `.msi`, `.msix`, `.msixbundle`, `.appx`, `.appxbundle`, `.nupkg`, `.dmg`, `.pkg`) and archives dload has no extractor for (known archive formats other than `.zip`, `.tar.gz`, `.tgz`, `.tar`, `.gz`, `.phar` — such as `.tar.xz`, `.tar.bz2`, `.zst`, `.pkg.tar.zst`, `.7z`, `.rar`) everywhere: dload cannot unpack them, so a release that has only such assets for the host gives way to an older one among the 10 newest releases that satisfy `version` (pin `version` to go further back). On ARM macOS and Windows, x86-64 builds are a fallback: Rosetta 2 and the Windows emulation run them. On macOS, a universal build — `universal`, `universal2` or `all` right next to `darwin` or `macos`, like `darwin-universal` or `macOS_all` — counts as native on both Intel and Apple Silicon, so it is kept by a strict selection and ranks with the host-arch build; elsewhere it has no arch. +Build variants stay in the pattern too. Among the assets for the host, dload prefers the lowest x86-64 microarchitecture level (`amd64` or `amd64_v1` over `amd64_v2`, `_v3`, `_v4`, also after `x86_64` or `x86-64`: the CPU is not checked, so a higher level is only a fallback, still picked when it is the only build for the host), then the host libc (`musl` builds on Alpine and Android, the others elsewhere), then the name with the fewest extra tokens — so `bun-linux-x64.zip` wins over `-baseline`, `-profile` and `-debug` twins — then archives. Android builds are dropped on Linux whenever the entry has a `binary` (without one, every other platform is only ranked lower), and so are OS packages (`.deb`, `.rpm`, `.apk`, `.snap`, `.flatpak`, `.msi`, `.msix`, `.msixbundle`, `.appx`, `.appxbundle`, `.nupkg`, `.dmg`, `.pkg`) and archives dload has no extractor for (known archive formats other than `.zip`, `.tar.gz`, `.tgz`, `.tar`, `.gz`, `.phar` — such as `.tar.xz`, `.tar.bz2`, `.zst`, `.pkg.tar.zst`, `.7z`, `.rar`) everywhere: dload cannot unpack them, so a release that has only such assets for the host gives way to an older one among the 10 newest releases that satisfy `version` (pin `version` to go further back). On ARM macOS and Windows, x86-64 builds are a fallback: Rosetta 2 and the Windows emulation run them. On macOS, a universal build — `universal`, `universal2`, `universal_binary` or `all` right next to `darwin`, `macos` or `osx`, like `darwin-universal`, `osx-universal_binary` or `macOS_all` — runs natively on both Intel and Apple Silicon, so it is kept by a strict selection and ranks right after the host-arch build, ahead of an x86-64 build under Rosetta; elsewhere it has no arch. Tokens the OS/arch matchers recognise (case-insensitive, bounded by `_` or a word boundary): | Kind | Tokens | |---|---| -| OS | `windows`, `linux`, `darwin`, `macos`, `alpine`, `bsd`, `freebsd`, `win32`, `win64`, `android` (wins over `linux`) | -| Arch | `amd64`, `arm64`, `aarch64`, `x86_64`, `x86-64`, `x64`, `win64` | +| OS | `windows`, `linux`, `darwin`, `macos`, `osx`, `alpine`, `bsd`, `freebsd`, `win32`, `win64`, `android` (wins over `linux`) | +| Arch | `amd64`, `arm64`, `aarch64`, `aarch_64`, `aarch-64`, `x86_64`, `x86-64`, `x64`, `win64` | | Libc | `musl`, `musleabi*`, `alpine`; `gnu`, `gnueabi*`, `glibc`; none means glibc | | Situation | Pattern | diff --git a/skills/dload-fetch-tool/references/troubleshooting.md b/skills/dload-fetch-tool/references/troubleshooting.md index 23026d6..b951782 100644 --- a/skills/dload-fetch-tool/references/troubleshooting.md +++ b/skills/dload-fetch-tool/references/troubleshooting.md @@ -37,12 +37,12 @@ curl -s "https://api.github.com/repos///releases/tags/" \ After filtering, dload picks the host's variant with: -- OS: `/(?:\b|_)(windows|linux|darwin|macos|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i`; an `android` token makes it Android, a bare `musl` makes it Linux -- Arch: `/(?:\b|_)(amd64|arm64|aarch64|x86[_-]64|x64|win64)(?:\b|_)/i` +- OS: `/(?:\b|_)(windows|linux|darwin|macos|osx|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i`; an `android` token makes it Android, a bare `musl` makes it Linux +- Arch: `/(?:\b|_)(amd64|arm64|aarch[_-]?64|x86[_-]64|x64|win64)(?:\b|_)/i` A candidate that matches neither is discarded. Common offenders: -- Non-standard tokens: `linux64`, `osx`. A universal macOS build (`darwin-universal`, `macOS_all`) has no arch token but is kept on macOS: `universal`, `universal2` or `all` next to `darwin`/`macos` counts as the host arch. +- Non-standard tokens: `linux64`, `macosx`. A universal macOS build (`darwin-universal`, `osx-universal_binary`, `macOS_all`) has no arch token but is kept on macOS: `universal`, `universal2`, `universal_binary` or `all` next to `darwin`/`macos`/`osx` runs on the host arch, ranked right after the host-arch build. - No arch separator: `tool-linux64.tar.gz` — the OS matches, no arch token is extractable. Compare with the host: `php -r "echo PHP_OS_FAMILY, ' / ', php_uname('m'), PHP_EOL;"`. diff --git a/src/Module/Common/Architecture.php b/src/Module/Common/Architecture.php index 77eb259..abb33a9 100644 --- a/src/Module/Common/Architecture.php +++ b/src/Module/Common/Architecture.php @@ -45,14 +45,14 @@ public static function tryFromString(string $arch): ?self { return match ($arch) { 'AMD64', 'amd64', 'x64', 'x86_64', 'x86-64', 'win64' => self::X86_64, - 'arm64', 'aarch64' => self::ARM_64, + 'arm64', 'aarch64', 'aarch_64', 'aarch-64' => self::ARM_64, default => null, }; } public static function tryFromBuildName(string $name): ?self { - if (\preg_match('/(?:\b|_)(amd64|arm64|aarch64|x86[_-]64|x64|win64)(?:\b|_)/i', $name, $matches) !== 1) { + if (\preg_match('/(?:\b|_)(amd64|arm64|aarch[_-]?64|x86[_-]64|x64|win64)(?:\b|_)/i', $name, $matches) !== 1) { return null; } diff --git a/src/Module/Common/OperatingSystem.php b/src/Module/Common/OperatingSystem.php index c7f1e5b..d446f63 100644 --- a/src/Module/Common/OperatingSystem.php +++ b/src/Module/Common/OperatingSystem.php @@ -64,7 +64,7 @@ public static function tryFromString(string $name): ?self return match (\strtolower($name)) { 'windows', 'win32', 'win64' => self::Windows, 'bsd', 'freebsd' => self::BSD, - 'darwin', 'macos' => self::Darwin, + 'darwin', 'macos', 'osx' => self::Darwin, 'android' => self::Android, // The libc is a separate trait, see {@see Libc} 'linux', 'alpine', 'unknown-musl' => self::Linux, @@ -80,7 +80,7 @@ public static function tryFromBuildName(string $name): ?self } if (\preg_match( - '/(?:\b|_)(windows|linux|darwin|macos|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i', + '/(?:\b|_)(windows|linux|darwin|macos|osx|alpine|bsd|freebsd|win32|win64)(?:\b|_)/i', $name, $matches, ) === 1) { diff --git a/src/Module/Downloader/Internal/AssetSelection/AssetName.php b/src/Module/Downloader/Internal/AssetSelection/AssetName.php index 93ac748..e8a5287 100644 --- a/src/Module/Downloader/Internal/AssetSelection/AssetName.php +++ b/src/Module/Downloader/Internal/AssetSelection/AssetName.php @@ -22,14 +22,14 @@ final class AssetName /** * Platform tokens: OS, architecture, libc and the vendor part of target triples. */ - private const PLATFORM = '/(?rank( self::KEY, - fn(Candidate $candidate): int => $this->rank($candidate) ?? 2, + fn(Candidate $candidate): int => $this->rank($candidate) ?? 3, )); } /** - * @return int<0, 1>|null Null when the host cannot run the asset. + * @return int<0, 2>|null Null when the host cannot run the asset. */ private function rank(Candidate $candidate): ?int { $arch = $candidate->asset->getArchitecture(); return match (true) { $arch === $this->architecture => 0, - $candidate->name->universal && $this->operatingSystem === OperatingSystem::Darwin => 0, + $candidate->name->universal && $this->operatingSystem === OperatingSystem::Darwin => self::UNIVERSAL, // Rosetta 2 on macOS and the built-in emulation of Windows run x86-64 builds on ARM $arch === Architecture::X86_64 && $this->architecture === Architecture::ARM_64 diff --git a/tests/Unit/Module/Common/ArchitectureTest.php b/tests/Unit/Module/Common/ArchitectureTest.php index df78c23..cc16248 100644 --- a/tests/Unit/Module/Common/ArchitectureTest.php +++ b/tests/Unit/Module/Common/ArchitectureTest.php @@ -23,6 +23,10 @@ public static function provideBuildNames(): iterable yield ['temporal-test-server_1.33.0_macOS_arm64.tar.gz', Architecture::ARM_64]; yield ['tool_1.0.0_linux_amd64_v3.tar.gz', Architecture::X86_64]; yield ['tool-linux-x86-64-v3.tar.gz', Architecture::X86_64]; + yield ['protoc-36.2-linux-aarch_64.zip', Architecture::ARM_64]; + yield ['protoc-36.2-osx-aarch_64.zip', Architecture::ARM_64]; + yield ['protoc-36.2-osx-x86_64.zip', Architecture::X86_64]; + yield ['protoc-36.2-osx-universal_binary.zip', null]; } #[DataProvider('provideBuildNames')] diff --git a/tests/Unit/Module/Common/OperatingSystemTest.php b/tests/Unit/Module/Common/OperatingSystemTest.php index 3ff34d9..aa66201 100644 --- a/tests/Unit/Module/Common/OperatingSystemTest.php +++ b/tests/Unit/Module/Common/OperatingSystemTest.php @@ -25,6 +25,8 @@ public static function provideBuildNames(): iterable yield ['temporal-test-server_1.33.0_macOS_arm64.tar.gz', OperatingSystem::Darwin]; yield ['bun-linux-x64-android-baseline.zip', OperatingSystem::Android]; yield ['tool-aarch64-linux-android.tar.gz', OperatingSystem::Android]; + yield ['protoc-36.2-osx-aarch_64.zip', OperatingSystem::Darwin]; + yield ['protoc-36.2-osx-universal_binary.zip', OperatingSystem::Darwin]; } public static function provideHosts(): iterable @@ -54,6 +56,12 @@ public function androidCanBeRequestedByName(): void Assert::same(OperatingSystem::tryFromString('android'), OperatingSystem::Android); } + #[Test] + public function osxIsReadAsDarwin(): void + { + Assert::same(OperatingSystem::tryFromString('osx'), OperatingSystem::Darwin); + } + #[Test] public function anUnknownHostFamilyIsRejected(): void { diff --git a/tests/Unit/Module/Downloader/AssetSelectionTest.php b/tests/Unit/Module/Downloader/AssetSelectionTest.php index 6414b39..5d40229 100644 --- a/tests/Unit/Module/Downloader/AssetSelectionTest.php +++ b/tests/Unit/Module/Downloader/AssetSelectionTest.php @@ -127,6 +127,27 @@ final class AssetSelectionTest 'vortex-driver-zig-x86_64-linux.zip', ]; + /** + * Assets of the Protocol Buffers v36.2 release: `osx` for macOS and `aarch_64` for arm64. + */ + private const PROTOC_ASSETS = [ + 'protobuf-36.2.bazel.tar.gz', + 'protobuf-36.2.bazel.tar.gz.intoto.jsonl', + 'protobuf-36.2.tar.gz', + 'protobuf-36.2.zip', + 'protoc-36.2-linux-aarch_64.zip', + 'protoc-36.2-linux-ppcle_64.zip', + 'protoc-36.2-linux-s390_64.zip', + 'protoc-36.2-linux-x86_32.zip', + 'protoc-36.2-linux-x86_64.zip', + 'protoc-36.2-osx-aarch_64.zip', + 'protoc-36.2-osx-universal_binary.zip', + 'protoc-36.2-osx-x86_64.zip', + 'protoc-36.2-win32.zip', + 'protoc-36.2-win64.zip', + 'tool_integrity.bzl', + ]; + private string $tempDir; public static function provideBunHosts(): \Generator @@ -162,6 +183,15 @@ public static function provideMagoHosts(): \Generator yield 'Linux arm64 musl' => [Architecture::ARM_64, Libc::Musl, 'mago-1.51.2-aarch64-unknown-linux-musl.tar.gz']; } + public static function provideProtocHosts(): \Generator + { + yield 'Linux x64' => [OperatingSystem::Linux, Architecture::X86_64, 'protoc-36.2-linux-x86_64.zip']; + yield 'Linux arm64' => [OperatingSystem::Linux, Architecture::ARM_64, 'protoc-36.2-linux-aarch_64.zip']; + yield 'macOS x64' => [OperatingSystem::Darwin, Architecture::X86_64, 'protoc-36.2-osx-x86_64.zip']; + yield 'macOS arm64' => [OperatingSystem::Darwin, Architecture::ARM_64, 'protoc-36.2-osx-aarch_64.zip']; + yield 'Windows x64' => [OperatingSystem::Windows, Architecture::X86_64, 'protoc-36.2-win64.zip']; + } + #[DataProvider('provideBunHosts')] #[Test] public function bunRegistryEntrySelectsThePlainBuild( @@ -248,6 +278,39 @@ public function magoBuildForTheHostLibcIsSelected(Architecture $arch, Libc $libc Assert::same($result->file->getFilename(), $expected); } + #[DataProvider('provideProtocHosts')] + #[Test] + public function protocBuildForTheHostIsSelected(OperatingSystem $os, Architecture $arch, string $expected): void + { + $result = $this->download( + self::registryEntry('protoc'), + 'protocolbuffers/protobuf', + 'v36.2', + self::PROTOC_ASSETS, + $os, + $arch, + Libc::Gnu, + ); + + Assert::same($result->file->getFilename(), $expected); + } + + #[Test] + public function protocUniversalBuildIsSelectedWithoutAnArmBuild(): void + { + $result = $this->download( + self::registryEntry('protoc'), + 'protocolbuffers/protobuf', + 'v36.2', + ['protoc-36.2-linux-aarch_64.zip', 'protoc-36.2-osx-universal_binary.zip', 'protoc-36.2-osx-x86_64.zip'], + OperatingSystem::Darwin, + Architecture::ARM_64, + Libc::Gnu, + ); + + Assert::same($result->file->getFilename(), 'protoc-36.2-osx-universal_binary.zip'); + } + #[BeforeTest] protected function setUp(): void { diff --git a/tests/Unit/Module/Downloader/Internal/AssetSelection/AssetNameTest.php b/tests/Unit/Module/Downloader/Internal/AssetSelection/AssetNameTest.php index 96f291b..183a0b7 100644 --- a/tests/Unit/Module/Downloader/Internal/AssetSelection/AssetNameTest.php +++ b/tests/Unit/Module/Downloader/Internal/AssetSelection/AssetNameTest.php @@ -29,6 +29,8 @@ public static function provideExtras(): iterable yield 'all macOS architectures' => ['tool_1.0.0_macOS_all.tar.gz', ['tool']]; yield 'universal not next to macOS' => ['universal-ctags-linux-x64.tar.gz', ['universal', 'ctags']]; yield 'microarchitecture level' => ['tool_1.0.0_linux_amd64_v3.tar.gz', ['tool']]; + yield 'osx and aarch_64' => ['protoc-36.2-osx-aarch_64.zip', ['protoc']]; + yield 'universal binary' => ['protoc-36.2-osx-universal_binary.zip', ['protoc']]; } public static function provideUniversal(): iterable @@ -37,6 +39,8 @@ public static function provideUniversal(): iterable yield 'universal before macOS' => ['tigerbeetle-universal-macos.zip', true]; yield 'macOS_all' => ['tool_1.0.0_macOS_all.tar.gz', true]; yield 'universal2' => ['tool-macos-universal2.zip', true]; + yield 'osx-universal_binary' => ['protoc-36.2-osx-universal_binary.zip', true]; + yield 'osx-universal' => ['tool-osx-universal.tar.gz', true]; yield 'single architecture' => ['tool-darwin-arm64.tar.gz', false]; yield 'all without macOS' => ['tool-all-linux-amd64.tar.gz', false]; yield 'universal without macOS' => ['universal-ctags-linux-x64.tar.gz', false]; diff --git a/tests/Unit/Module/Downloader/Internal/AssetSelection/Rule/ArchitectureRuleTest.php b/tests/Unit/Module/Downloader/Internal/AssetSelection/Rule/ArchitectureRuleTest.php index 6293514..22ded64 100644 --- a/tests/Unit/Module/Downloader/Internal/AssetSelection/Rule/ArchitectureRuleTest.php +++ b/tests/Unit/Module/Downloader/Internal/AssetSelection/Rule/ArchitectureRuleTest.php @@ -30,8 +30,16 @@ public static function provideEmulatingHosts(): iterable public static function provideMacArchitectures(): iterable { - yield 'Apple Silicon' => [Architecture::ARM_64, ArchitectureRule::EMULATED]; - yield 'Intel' => [Architecture::X86_64, 0]; + yield 'Apple Silicon' => [Architecture::ARM_64, [ + 'tool-darwin-universal.zip' => ArchitectureRule::UNIVERSAL, + 'tool-darwin-x86_64.zip' => ArchitectureRule::EMULATED, + 'tool-darwin-aarch64.zip' => 0, + ]]; + yield 'Intel' => [Architecture::X86_64, [ + 'tool-darwin-universal.zip' => ArchitectureRule::UNIVERSAL, + 'tool-darwin-x86_64.zip' => 0, + 'tool-darwin-aarch64.zip' => 3, + ]]; } #[Test] @@ -52,9 +60,9 @@ public function aGradualSelectionRanksOtherArchitecturesLast(): void ); Assert::same(RuleRunner::ranks($selection, 'arch'), [ - 'tool-x86_64.zip' => 2, + 'tool-x86_64.zip' => 3, 'tool-aarch64.zip' => 0, - 'tool.zip' => 2, + 'tool.zip' => 3, ]); } @@ -65,7 +73,7 @@ public function armHostsThatEmulateX86KeepItsBuildsAsAFallback(OperatingSystem $ $selection = RuleRunner::run(new ArchitectureRule(Architecture::ARM_64, $os), self::ASSETS); Assert::same(RuleRunner::ranks($selection, 'arch'), [ - 'tool-x86_64.zip' => 1, + 'tool-x86_64.zip' => ArchitectureRule::EMULATED, 'tool-aarch64.zip' => 0, ]); } @@ -79,19 +87,20 @@ public function x86HostsDoNotRunArmBuilds(OperatingSystem $os): void Assert::same(RuleRunner::ranks($selection, 'arch'), ['tool-x86_64.zip' => 0]); } + /** + * @param array $expected + */ #[DataProvider('provideMacArchitectures')] #[Test] - public function aUniversalBuildIsNativeOnMacOs(Architecture $arch, int $x86Rank): void + public function aUniversalBuildRanksRightAfterTheHostBuildOnMacOs(Architecture $arch, array $expected): void { $selection = RuleRunner::run(new ArchitectureRule($arch, OperatingSystem::Darwin), [ 'tool-darwin-universal.zip', 'tool-darwin-x86_64.zip', - ]); + 'tool-darwin-aarch64.zip', + ], strict: false); - Assert::same(RuleRunner::ranks($selection, 'arch'), [ - 'tool-darwin-universal.zip' => 0, - 'tool-darwin-x86_64.zip' => $x86Rank, - ]); + Assert::same(RuleRunner::ranks($selection, 'arch'), $expected); } #[Test] @@ -102,7 +111,7 @@ public function aUniversalMacOsBuildHasNoArchitectureElsewhere(): void Assert::same(RuleRunner::ranks(RuleRunner::run($rule, $assets), 'arch'), ['tool-linux-x86_64.zip' => 0]); Assert::same(RuleRunner::ranks(RuleRunner::run($rule, $assets, strict: false), 'arch'), [ - 'tool-darwin-universal.zip' => 2, + 'tool-darwin-universal.zip' => 3, 'tool-linux-x86_64.zip' => 0, ]); }