diff --git a/.github/workflows/python-native-qualification.yml b/.github/workflows/python-native-qualification.yml index ac60e9d..0e8026c 100644 --- a/.github/workflows/python-native-qualification.yml +++ b/.github/workflows/python-native-qualification.yml @@ -45,11 +45,11 @@ jobs: python-version: ${{ matrix.python }} - name: Check candidate package metadata run: | - python -c "import tomllib; from pathlib import Path; project=tomllib.loads(Path('bindings/python/pyproject.toml').read_text())['project']; binding=tomllib.loads(Path('bindings/python/Cargo.toml').read_text())['package']; root=tomllib.loads(Path('Cargo.toml').read_text())['package']; assert project['name'] == 'eggsearch'; assert project['license'] == 'MIT'; assert project['version'] == binding['version'] == root['version']; print(project['name'], project['version'], project['license'])" + python -c "import tomllib; from pathlib import Path; project=tomllib.loads(Path('bindings/python/pyproject.toml').read_text())['project']; binding=tomllib.loads(Path('bindings/python/Cargo.toml').read_text())['package']; root=tomllib.loads(Path('Cargo.toml').read_text())['package']; assert project['name'] == 'eggsearch'; assert project['license'] == 'MIT'; assert project['version'] == binding['version'] == root['version']; assert project['requires-python'] == '>=3.11,<3.16'; assert all(f'Programming Language :: Python :: 3.{minor}' in project['classifiers'] for minor in range(11,16)); assert 'abi3-py311' in Path('bindings/python/Cargo.toml').read_text(); print(project['name'], project['version'], project['requires-python'], project['license'])" - if: matrix.linux uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b with: - maturin-version: v1.9.5 + maturin-version: v1.15.0 rust-toolchain: 1.89.0 manylinux: "2014" args: --manifest-path bindings/python/Cargo.toml --release --locked --out dist -i python${{ matrix.python }} @@ -59,14 +59,15 @@ jobs: toolchain: 1.89.0 targets: ${{ matrix.target }} - if: ${{ !matrix.linux }} - run: python -m pip install maturin==1.9.5 + run: python -m pip install maturin==1.15.0 - if: ${{ !matrix.linux }} run: maturin build --manifest-path bindings/python/Cargo.toml --release --locked --out dist --interpreter python - name: Record wheel metadata and digest shell: bash run: | + python -m pip install packaging==26.0 python -m pip install --no-index --no-deps --only-binary=:all: dist/*.whl - python -c "from importlib.metadata import distribution; d=distribution('eggsearch'); print(d.metadata['Name'], d.version); print(*d.files, sep='\\n')" > dist/metadata.txt + python -c "from importlib.metadata import distribution; from packaging.utils import parse_wheel_filename; from pathlib import Path; d=distribution('eggsearch'); wheels=list(Path('dist').glob('*.whl')); assert len(wheels) == 1; name, version, build, tags=parse_wheel_filename(wheels[0].name); assert name == 'eggsearch' and str(version) == d.version; assert all(tag.abi == 'abi3' and tag.interpreter == 'cp311' for tag in tags); print(d.metadata['Name'], d.version, d.metadata['Requires-Python']); print(*d.files, sep='\\n')" > dist/metadata.txt sha256sum dist/*.whl | tee dist/SHA256SUMS cat dist/SHA256SUMS >> "$GITHUB_STEP_SUMMARY" - id: upload @@ -93,22 +94,27 @@ jobs: - { id: linux-x86_64, runner: ubuntu-24.04, target: x86_64-unknown-linux-gnu, python: "3.12" } - { id: linux-x86_64, runner: ubuntu-24.04, target: x86_64-unknown-linux-gnu, python: "3.13" } - { id: linux-x86_64, runner: ubuntu-24.04, target: x86_64-unknown-linux-gnu, python: "3.14" } + - { id: linux-x86_64, runner: ubuntu-24.04, target: x86_64-unknown-linux-gnu, python: "3.15" } - { id: linux-aarch64, runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, python: "3.11" } - { id: linux-aarch64, runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, python: "3.12" } - { id: linux-aarch64, runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, python: "3.13" } - { id: linux-aarch64, runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, python: "3.14" } + - { id: linux-aarch64, runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, python: "3.15" } - { id: macos-x86_64, runner: macos-15-intel, target: x86_64-apple-darwin, python: "3.11" } - { id: macos-x86_64, runner: macos-15-intel, target: x86_64-apple-darwin, python: "3.12" } - { id: macos-x86_64, runner: macos-15-intel, target: x86_64-apple-darwin, python: "3.13" } - { id: macos-x86_64, runner: macos-15-intel, target: x86_64-apple-darwin, python: "3.14" } + - { id: macos-x86_64, runner: macos-15-intel, target: x86_64-apple-darwin, python: "3.15" } - { id: macos-arm64, runner: macos-15, target: aarch64-apple-darwin, python: "3.11" } - { id: macos-arm64, runner: macos-15, target: aarch64-apple-darwin, python: "3.12" } - { id: macos-arm64, runner: macos-15, target: aarch64-apple-darwin, python: "3.13" } - { id: macos-arm64, runner: macos-15, target: aarch64-apple-darwin, python: "3.14" } + - { id: macos-arm64, runner: macos-15, target: aarch64-apple-darwin, python: "3.15" } - { id: windows-x86_64, runner: windows-2022, target: x86_64-pc-windows-msvc, python: "3.11" } - { id: windows-x86_64, runner: windows-2022, target: x86_64-pc-windows-msvc, python: "3.12" } - { id: windows-x86_64, runner: windows-2022, target: x86_64-pc-windows-msvc, python: "3.13" } - { id: windows-x86_64, runner: windows-2022, target: x86_64-pc-windows-msvc, python: "3.14" } + - { id: windows-x86_64, runner: windows-2022, target: x86_64-pc-windows-msvc, python: "3.15" } steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 @@ -134,6 +140,67 @@ jobs: shell: bash working-directory: ${{ runner.temp }} + armv7-build: + name: Build ARMv7 hard-float wheel + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b + with: + maturin-version: v1.15.0 + rust-toolchain: 1.89.0 + target: armv7-unknown-linux-gnueabihf + manylinux: "2_28" + container: ghcr.io/rust-cross/manylinux_2_28-cross@sha256:b1f069c62f7744faa6db5f1ab28b11d59f72e161d8f03f6b0a388fe1cecc065d + args: --manifest-path bindings/python/Cargo.toml --release --locked --out dist -i python3.11 --compatibility pypi + - name: Verify ARMv7 wheel metadata + run: | + python -m pip install packaging==26.0 + python bindings/python/scripts/audit_armv7_wheel.py dist/*.whl | tee dist/armv7-audit.txt + sha256sum dist/*.whl | tee dist/SHA256SUMS + cat dist/armv7-audit.txt >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: wheel-linux-armv7l + path: dist/ + if-no-files-found: error + retention-days: 14 + + armv7-wheel-smoke: + name: Emulated ARMv7 installed-wheel smoke + needs: armv7-build + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a + with: + platforms: arm + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: wheel-linux-armv7l + path: ${{ runner.temp }}/wheelhouse + - name: Install and test only the prebuilt wheel in matching ARMv7 Linux + shell: bash + run: | + docker run --rm --platform linux/arm/v7 \ + -v "$RUNNER_TEMP/wheelhouse:/wheelhouse:ro" \ + -v "$GITHUB_WORKSPACE:/source:ro" \ + quay.io/pypa/manylinux_2_31_armv7l@sha256:ff4a72812594311c071eebce43efcaa9061bd1205649d864c1054d4e56e6bdae \ + /bin/bash -euxo pipefail -c ' + for tag in cp311 cp315; do + python_bin="/opt/python/$tag-$tag/bin/python" + test -x "$python_bin" + venv="/tmp/eggsearch-$tag-venv" + "$python_bin" -m venv "$venv" + "$venv/bin/python" -m pip install --no-index --no-deps --only-binary=:all: /wheelhouse/*.whl + "$venv/bin/python" -m pip install pytest==8.4.2 + cd /tmp + "$venv/bin/python" -m pytest -q /source/bindings/python/tests + "$venv/bin/python" /source/bindings/python/scripts/smoke_installed_wheel.py + "$venv/bin/python" -c "import platform,sys; from importlib.metadata import distribution; import eggsearch; assert platform.machine() in (\"armv7l\", \"armv8l\"); assert sys.version_info[:2] == (3, int(\"$tag\"[3:])); assert len(eggsearch.tool_names()) == 10; print(platform.machine(), sys.version, distribution(\"eggsearch\").version)" + done + ' + source-distribution: name: Rebuild and smoke extracted sdist runs-on: ubuntu-24.04 @@ -144,8 +211,8 @@ jobs: toolchain: 1.89.0 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: "3.14" - - run: python -m pip install maturin==1.9.5 + python-version: "3.15" + - run: python -m pip install maturin==1.15.0 - name: Build sdist, prefetch locked crates, and rebuild offline shell: bash run: | @@ -161,16 +228,15 @@ jobs: "$RUNNER_TEMP/runtime-venv/bin/python" -m pip install pytest==8.4.2 cd "$RUNNER_TEMP" "$RUNNER_TEMP/runtime-venv/bin/python" -m pytest -q "$source_dir/bindings/python/tests" - "$RUNNER_TEMP/runtime-venv/bin/python" -c "import eggsearch; from importlib.metadata import distribution; assert len(eggsearch.tool_names()) == 10; assert distribution('eggsearch').version == '0.4.4'" + "$RUNNER_TEMP/runtime-venv/bin/python" -c "import tomllib; from pathlib import Path; import eggsearch; from importlib.metadata import distribution; expected=tomllib.loads(Path('$GITHUB_WORKSPACE/bindings/python/pyproject.toml').read_text())['project']['version']; assert len(eggsearch.tool_names()) == 10; assert distribution('eggsearch').version == expected" sha256sum "$GITHUB_WORKSPACE"/dist/* "$GITHUB_WORKSPACE"/extracted-wheel/* | tee "$GITHUB_WORKSPACE/dist/SHA256SUMS" cat "$GITHUB_WORKSPACE/dist/SHA256SUMS" >> "$GITHUB_STEP_SUMMARY" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: - name: python-sdist-and-extracted-wheel + name: python-sdist-qualification path: | dist/*.tar.gz dist/SHA256SUMS - extracted-wheel/*.whl if-no-files-found: error retention-days: 14 @@ -183,11 +249,12 @@ jobs: with: toolchain: 1.89.0 - run: cargo install cargo-deny --version 0.20.2 --locked + - run: cargo deny check - run: cargo deny check working-directory: bindings/python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: python-version: "3.14" - run: python -m pip install --upgrade pip==26.2.1 - - run: python -m pip install pip-audit==2.10.1 maturin==1.9.5 + - run: python -m pip install pip-audit==2.10.1 maturin==1.15.0 - run: pip-audit --local --progress-spinner off diff --git a/.github/workflows/release-python-pypi.yml b/.github/workflows/release-python-pypi.yml new file mode 100644 index 0000000..0cb3f0c --- /dev/null +++ b/.github/workflows/release-python-pypi.yml @@ -0,0 +1,178 @@ +name: Python PyPI release + +on: + workflow_dispatch: + inputs: + candidate_sha: + description: Exact 40-character commit SHA that passed qualification + required: true + type: string + qualification_run_id: + description: Successful Python native qualification workflow run ID + required: true + type: string + version: + description: Exact synchronized package version + required: true + type: string + +permissions: + contents: read + +jobs: + verify-candidate: + name: Verify qualified source and artifact inventory + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ inputs.candidate_sha }} + persist-credentials: false + - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 + with: + toolchain: 1.89.0 + - name: Verify exact candidate and successful qualification runs + env: + GH_TOKEN: ${{ github.token }} + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + QUALIFICATION_RUN_ID: ${{ inputs.qualification_run_id }} + RELEASE_VERSION: ${{ inputs.version }} + run: | + test "${#CANDIDATE_SHA}" -eq 40 + case "$CANDIDATE_SHA" in *[!0-9a-f]*) exit 1 ;; esac + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + python -c "import os,tomllib; from pathlib import Path; expected=os.environ['RELEASE_VERSION']; root=tomllib.loads(Path('Cargo.toml').read_text())['package']['version']; binding=tomllib.loads(Path('bindings/python/Cargo.toml').read_text())['package']['version']; project=tomllib.loads(Path('bindings/python/pyproject.toml').read_text())['project']['version']; assert root == binding == project == expected" + qualification_sha="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$QUALIFICATION_RUN_ID" --jq 'select(.name == "Python native qualification" and .status == "completed" and .conclusion == "success") | .head_sha')" + test "$qualification_sha" = "$CANDIDATE_SHA" + ci_runs="$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?head_sha=$CANDIDATE_SHA&per_page=100" --jq "[.workflow_runs[] | select(.name == \"CI\" and .status == \"completed\" and .conclusion == \"success\" and .head_sha == \"$CANDIDATE_SHA\")] | length")" + test "$ci_runs" -ge 1 + test "$(gh api "repos/$GITHUB_REPOSITORY/environments/pypi" --jq '.name')" = "pypi" + cargo publish --dry-run --locked + pypi_status="$(curl --silent --output /dev/null --write-out '%{http_code}' "https://pypi.org/pypi/eggsearch/$RELEASE_VERSION/json")" + test "$pypi_status" = 404 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.15" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + run-id: ${{ inputs.qualification_run_id }} + github-token: ${{ github.token }} + path: artifacts + - name: Verify immutable release inventory + env: + RELEASE_VERSION: ${{ inputs.version }} + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + QUALIFICATION_RUN_ID: ${{ inputs.qualification_run_id }} + run: | + python -m pip install packaging==26.0 + python bindings/python/scripts/verify_release_artifacts.py artifacts "$RELEASE_VERSION" release-manifest.json --source-sha "$CANDIDATE_SHA" --qualification-run-id "$QUALIFICATION_RUN_ID" + mkdir -p release/packages + find artifacts -type f \( -name '*.whl' -o -name '*.tar.gz' \) -exec cp '{}' release/packages/ \; + cp release-manifest.json release/MANIFEST.json + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: verified-python-release-inventory + path: release/ + if-no-files-found: error + retention-days: 14 + + publish: + name: Manually dispatched PyPI publication + needs: verify-candidate + runs-on: ubuntu-24.04 + environment: + name: pypi + url: https://pypi.org/project/eggsearch/ + permissions: + actions: read + id-token: write + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: verified-python-release-inventory + path: release + - name: Publish verified inventory with PyPI Trusted Publishing + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 + with: + packages-dir: release/packages/ + + post-publication-consumer: + name: Public PyPI install on ${{ matrix.platform }} CPython 3.15 + needs: publish + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { platform: linux-x86_64, runner: ubuntu-24.04 } + - { platform: linux-aarch64, runner: ubuntu-24.04-arm } + - { platform: macos-x86_64, runner: macos-15-intel } + - { platform: macos-arm64, runner: macos-15 } + - { platform: windows-x86_64, runner: windows-2022 } + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ inputs.candidate_sha }} + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.15" + - name: Install only the published binary wheel + env: + RELEASE_VERSION: ${{ inputs.version }} + run: | + python -m venv "$RUNNER_TEMP/consumer-venv" + if [ "$RUNNER_OS" = "Windows" ]; then + python_bin="$(cygpath -u "$RUNNER_TEMP")/consumer-venv/Scripts/python.exe" + else + python_bin="$RUNNER_TEMP/consumer-venv/bin/python" + fi + "$python_bin" -m pip install --report "$RUNNER_TEMP/pip-report.json" --only-binary=:all: --no-deps "eggsearch==$RELEASE_VERSION" + "$python_bin" -m pip debug --verbose > "$RUNNER_TEMP/pip-debug.txt" + "$python_bin" -c "import json,os; report=json.load(open(os.path.join(os.environ['RUNNER_TEMP'],'pip-report.json'))); item=next(x for x in report['install'] if x['metadata']['name'].lower() == 'eggsearch'); print(item['download_info']['url'], item['download_info']['archive_info'].get('hashes',{}))" + cat "$RUNNER_TEMP/pip-debug.txt" >> "$GITHUB_STEP_SUMMARY" + echo "PYTHON_BIN=$python_bin" >> "$GITHUB_ENV" + shell: bash + - name: Smoke the installed public package outside the checkout + run: | + if [ "$RUNNER_OS" = "Windows" ]; then + workspace_path="$(cygpath -u "$GITHUB_WORKSPACE")" + else + workspace_path="$GITHUB_WORKSPACE" + fi + "$PYTHON_BIN" "$workspace_path/bindings/python/scripts/smoke_installed_wheel.py" + working-directory: ${{ runner.temp }} + shell: bash + + post-publication-armv7-consumer: + name: Public PyPI install in emulated ARMv7 manylinux 2.31 + needs: publish + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ inputs.candidate_sha }} + persist-credentials: false + - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a + with: + platforms: arm + - name: Install only the published ARMv7 wheel and smoke it + env: + RELEASE_VERSION: ${{ inputs.version }} + run: | + docker run --rm --platform linux/arm/v7 \ + -e RELEASE_VERSION \ + -v "$GITHUB_WORKSPACE:/source:ro" \ + quay.io/pypa/manylinux_2_31_armv7l@sha256:ff4a72812594311c071eebce43efcaa9061bd1205649d864c1054d4e56e6bdae \ + /bin/bash -euxo pipefail -c ' + python_bin=/opt/python/cp311-cp311/bin/python + "$python_bin" -m venv /tmp/eggsearch-venv + /tmp/eggsearch-venv/bin/python -m pip install --report /tmp/pip-report.json --only-binary=:all: --no-deps "eggsearch==${RELEASE_VERSION}" + cd /tmp + /tmp/eggsearch-venv/bin/python -c "import json; report=json.load(open(\"/tmp/pip-report.json\")); item=next(x for x in report[\"install\"] if x[\"metadata\"][\"name\"].lower() == \"eggsearch\"); print(item[\"download_info\"][\"url\"], item[\"download_info\"][\"archive_info\"].get(\"hashes\", {}))" + /tmp/eggsearch-venv/bin/python -m pip debug --verbose + /tmp/eggsearch-venv/bin/python /source/bindings/python/scripts/smoke_installed_wheel.py + ' diff --git a/Cargo.lock b/Cargo.lock index 41e548d..ba0e813 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1325,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2011,7 +2011,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2953,7 +2953,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2982,9 +2982,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.14" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -3414,7 +3414,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3553,9 +3553,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -4035,14 +4035,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] diff --git a/architecture/python-library.md b/architecture/python-library.md index 2bcf712..e0a8550 100644 --- a/architecture/python-library.md +++ b/architecture/python-library.md @@ -19,6 +19,15 @@ superseded as the qualification basis by M008 advisory remediation and symlink containment at `plans/closure/python-native-library/008-status.md` (candidate `067468a`, fully-green run `38641009246`). +A separate **active PyPI distribution workstream** (not a reopening of M001–M008) +is registered at `plans/subsystems/python-pypi-distribution-roadmap.md`. +It prepares CPython 3.11–3.15 ABI3 support, hosted prebuilt ARMv7/SBC wheels, +first controlled PyPI publication, and **post-publication** physical-SBC +benchmarking. No device-local source build or native SBC benchmark is a +first-release prerequisite; compatible SBC installs must use the published +prebuilt wheel and are labeled experimental pending hardware evidence. + + ## Rust operation facade `src/sdk.rs` owns the narrow `EggsearchClient`, canonical `ToolName` mapping, @@ -64,15 +73,16 @@ extension alone is not evidence of a supported Python consumer. Wheel support is limited to targets with native install and runtime qualification, and publication requires separate authorization. -M004's candidate binding uses PyO3's `abi3-py311` stable ABI. Its release -matrix builds a manylinux2014 Linux x86_64/aarch64 wheel, macOS x86_64/arm64 -wheels, and a Windows x86_64 wheel, then installs each exact artifact on -CPython 3.11–3.14 in separate consumer jobs. These platform claims remain -conditional on the matching exact-candidate matrix jobs passing. An sdist +The active PyPI distribution candidate uses PyO3's `abi3-py311` stable ABI. +Its release matrix builds a manylinux2014 Linux x86_64/aarch64 wheel, macOS +x86_64/arm64 wheels, and a Windows x86_64 wheel, then installs each exact +artifact on CPython 3.11–3.15 in separate consumer jobs. These platform claims +remain conditional on the matching exact-candidate matrix jobs passing. An sdist rebuild uses the binding and root Cargo manifests from clean extraction; it can build offline with locked crates already cached, otherwise Cargo registry -access is required. PyPy, free-threaded CPython, ARMv7, and Windows ARM64 are -outside the support matrix. +access is required. PyPy, free-threaded CPython, and Windows ARM64 are outside +the support matrix. ARMv7 requires its separate experimental wheel and +emulated installed-wheel qualification before it may be listed as supported. The public Python `AsyncEggsearch` wrapper adds `async with`, `aclose()`, and the ten named keyword methods over the generic native `execute` call. Request diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index fc205d2..5cfc581 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -650,7 +650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1547,9 +1547,9 @@ checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "pyo3" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +checksum = "229ca13b8a15bb0d1487b81ab975bfcd97513416499a8312011b6036e42e0de4" dependencies = [ "libc", "once_cell", @@ -1561,9 +1561,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046" +checksum = "b877331c7817ccb6379e9d19a06993160141f1d0a386e55276128717df85e3fc" dependencies = [ "futures-channel", "futures-util", @@ -1575,18 +1575,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" +checksum = "70130e2b6dd1ee589a8788026a429561ca5c40fa7a2fa7914c2282eddae24700" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +checksum = "5e7ff73c555c286d212d4ce7e5fbda845a973c4f61bd5f160f5f2c3c2a04d1c4" dependencies = [ "libc", "pyo3-build-config", @@ -1594,26 +1594,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" +checksum = "c46d2e5862c7bb8207e3b639f713dda3c499933444784154c11d013c5a691b60" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "pyo3-macros-backend" -version = "0.29.3" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" +checksum = "38b89f066f2ae72a2aef52d1537ca2fcd104440ebfadee06e97aa90738a25b9b" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -1832,7 +1832,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1861,9 +1861,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.14" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -2219,7 +2219,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2294,9 +2294,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -2654,14 +2654,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index b73eed4..9180b3b 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -15,8 +15,8 @@ crate-type = ["cdylib"] [dependencies] eggsearch = { path = "../..", version = "=0.4.4" } futures = "0.3" -pyo3 = { version = "=0.29.3", features = ["abi3-py311", "extension-module", "macros"] } -pyo3-async-runtimes = { version = "=0.29.0", features = ["tokio-runtime"] } +pyo3 = { version = "=0.30.1", features = ["abi3-py311", "extension-module", "macros"] } +pyo3-async-runtimes = { version = "=0.30.0", features = ["tokio-runtime"] } serde_json = "1" tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "sync", "time"] } tokio-util = "0.7" diff --git a/bindings/python/LICENSE-PYTHON b/bindings/python/LICENSE-PYTHON new file mode 100644 index 0000000..08acf81 --- /dev/null +++ b/bindings/python/LICENSE-PYTHON @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 David Bowman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/bindings/python/PYTHON_README.md b/bindings/python/PYTHON_README.md index e0e1421..dc9ac81 100644 --- a/bindings/python/PYTHON_README.md +++ b/bindings/python/PYTHON_README.md @@ -5,12 +5,12 @@ fetch, repository, security, research, diagnostic, and evidence operations. It calls the shared Rust SDK in-process and does not start the MCP server. This package is not published to PyPI. To build and install it locally, use -Rust 1.89, CPython 3.11–3.14, and an active virtual environment: +Rust 1.89, CPython 3.11–3.15, and an active virtual environment: ```bash python -m venv .venv source .venv/bin/activate -python -m pip install -e 'bindings/python[dev]' maturin==1.9.5 +python -m pip install -e 'bindings/python[dev]' maturin==1.15.0 maturin develop --manifest-path bindings/python/Cargo.toml --locked python -m pytest -q bindings/python/tests pyright bindings/python/python/eggsearch bindings/python/examples diff --git a/bindings/python/benchmarks/sbc_runtime.py b/bindings/python/benchmarks/sbc_runtime.py new file mode 100644 index 0000000..3ef21a0 --- /dev/null +++ b/bindings/python/benchmarks/sbc_runtime.py @@ -0,0 +1,398 @@ +import argparse +import asyncio +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import hashlib +import os +import platform +import shutil +import statistics +import subprocess +import sys +import tempfile +import threading +import time +import tracemalloc +from importlib.metadata import distribution +from pathlib import Path +from zipfile import ZipFile +from resource import getrusage, RUSAGE_SELF + +from eggsearch import AsyncEggsearch, EggsearchError, tool_names + + +def percentile(values, fraction): + ordered = sorted(values) + return ordered[min(len(ordered) - 1, round((len(ordered) - 1) * fraction))] + + +def summarize(values): + return { + "count": len(values), + "median_ms": statistics.median(values), + "p95_ms": percentile(values, 0.95), + "min_ms": min(values), + "max_ms": max(values), + } + + +def memory_snapshot(): + values: dict[str, int | str] = {"max_rss_kib": getrusage(RUSAGE_SELF).ru_maxrss} + status = Path("/proc/self/status") + if status.exists(): + for line in status.read_text().splitlines(): + if line.startswith(("VmRSS:", "VmHWM:", "Threads:")): + key, value = line.split(":", 1) + values[key.lower()] = value.strip() + rollup = Path("/proc/self/smaps_rollup") + if rollup.exists(): + for line in rollup.read_text().splitlines(): + if line.startswith(("Rss:", "Pss:")): + key, value = line.split(":", 1) + values[key.lower()] = value.strip() + fd_dir = Path("/proc/self/fd") + if fd_dir.exists(): + values["open_fds"] = len(list(fd_dir.iterdir())) + return values + + +def system_snapshot(installed): + disk = shutil.disk_usage(Path.cwd()) + values = { + "uname": platform.uname()._asdict(), + "cpu_count": os.cpu_count(), + "libc": platform.libc_ver(), + "filesystem_total_bytes": disk.total, + "filesystem_free_bytes": disk.free, + "installed_package_bytes": sum( + Path(installed.locate_file(path)).stat().st_size + for path in installed.files or [] + if Path(installed.locate_file(path)).is_file() + ), + } + for path, key in ( + (Path("/etc/os-release"), "os_release"), + (Path("/proc/meminfo"), "memory_info"), + (Path("/proc/cpuinfo"), "cpu_info"), + (Path("/proc/swaps"), "swap_info"), + ): + if path.exists(): + values[key] = path.read_text(errors="replace") + if "memory_info" in values: + available = next( + (line.split()[1] for line in values["memory_info"].splitlines() if line.startswith("MemAvailable:")), + None, + ) + if available is not None: + values["memory_available_kib"] = int(available) + model = Path("/proc/device-tree/model") + if model.exists(): + values["device_model"] = model.read_bytes().rstrip(b"\0").decode(errors="replace") + thermal = Path("/sys/class/thermal") + if thermal.exists(): + values["thermal_zones"] = { + path.name: path.read_text().strip() + for zone in thermal.glob("thermal_zone*") + for path in (zone / "temp",) + if path.exists() + } + frequencies = {} + for path in Path("/sys/devices/system/cpu").glob("cpu[0-9]*/cpufreq/scaling_cur_freq"): + frequencies[path.parents[1].name] = path.read_text().strip() + if frequencies: + values["cpu_frequency_khz"] = frequencies + throttling = {} + for path in Path("/sys/devices/system/cpu").glob("cpu[0-9]*/thermal_throttle/*_throttle_count"): + throttling[str(path)] = path.read_text().strip() + if throttling: + values["thermal_throttle_counts"] = throttling + return values + + +def cold_import_ms(): + started = time.perf_counter_ns() + subprocess.run( + [sys.executable, "-c", "import eggsearch"], + check=True, + stdout=subprocess.DEVNULL, + ) + return (time.perf_counter_ns() - started) / 1_000_000 + + +def local_config(directory, fixture): + config = directory / "eggsearch.toml" + config.write_text( + "[search]\nmode = 'off'\ndefault_max_results = 100\n" + "max_results_cap = 100\nmax_query_chars = 512\ntimeout_ms = 8000\n" + "default_providers = ['duckduckgo']\nsanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\n\n" + "[fetch]\nenabled = true\ntimeout_ms = 8000\nmax_bytes = 2000000\n" + "max_chars_default = 12000\nmax_chars_cap = 50000\nredirect_limit = 5\n" + "allow_private_network = false\nallow_localhost = true\n" + "sanitize_output = true\n\n" + f"[local]\nenabled = true\nroots = ['{fixture.as_posix()}']\n", + encoding="utf-8", + ) + return config + + +class FixtureHandler(BaseHTTPRequestHandler): + fixture = "" + + def do_GET(self): + body = self.fixture.encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return None + + +def start_fetch_fixture(): + handler = type("BoundFixtureHandler", (FixtureHandler,), {"fixture": "local fetch fixture " * 1000}) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread, f"http://127.0.0.1:{server.server_port}/fixture.txt" + + +async def timed_search(client, index): + started = time.perf_counter_ns() + result = await client.repo_search(query=f"retry fixture {index}", include_local=True) + elapsed = (time.perf_counter_ns() - started) / 1_000_000 + local_cards(result) + return elapsed + + +def local_cards(result): + if not isinstance(result.get("groups"), list): + raise RuntimeError("local Mode-Off search returned no groups field") + cards = [ + card + for group in result["groups"] + for card in group.get("results", []) + ] + if not cards or any(card.get("trust") != "local_trusted" for card in cards): + raise RuntimeError("Mode-Off smoke did not return only trusted local results") + return cards + + +async def sequential_event_loop_sample(config, index): + client = AsyncEggsearch(str(config)) + try: + return await timed_search(client, index) + finally: + await client.aclose() + + +async def run_benchmarks(config, repeats): + results = {"search": {}, "cancellation": {}, "memory": {"before": memory_snapshot()}} + default_started = time.perf_counter_ns() + default_client = AsyncEggsearch() + results["default_client_init_ms"] = (time.perf_counter_ns() - default_started) / 1_000_000 + await default_client.aclose() + results["memory"]["after_default_client_close"] = memory_snapshot() + explicit_started = time.perf_counter_ns() + client = AsyncEggsearch(str(config)) + results["explicit_mode_off_client_init_ms"] = (time.perf_counter_ns() - explicit_started) / 1_000_000 + initialization_started = time.perf_counter_ns() + await timed_search(client, 0) + results["first_operation_ms"] = (time.perf_counter_ns() - initialization_started) / 1_000_000 + results["memory"]["initialized"] = memory_snapshot() + validation_started = time.perf_counter_ns() + try: + await client.web_search(query="") + except EggsearchError as error: + if not error.code: + raise RuntimeError("malformed-input error did not include a code") from error + results["malformed_input"] = { + "rejected": True, + "error_code": error.code, + "elapsed_ms": (time.perf_counter_ns() - validation_started) / 1_000_000, + } + else: + raise RuntimeError("malformed empty query was not rejected") + + for concurrency in (1, 2, 4, 8, 16, 32): + operations = max(repeats, concurrency) + durations = [] + for offset in range(0, operations, concurrency): + batch = range(offset, min(offset + concurrency, operations)) + durations.extend(await asyncio.gather(*(timed_search(client, i) for i in batch))) + results["search"][str(concurrency)] = summarize(durations) + results["memory"][f"after_{concurrency}"] = memory_snapshot() + + heavy_started = time.perf_counter_ns() + heavy_result = await client.repo_search( + query="retry fixture evidence", + include_local=True, + max_results=100, + max_per_group=100, + response_detail="diagnostic", + ) + heavy_cards = local_cards(heavy_result) + results["heavy_output"] = { + "elapsed_ms": (time.perf_counter_ns() - heavy_started) / 1_000_000, + "serialized_bytes": len(json.dumps(heavy_result, separators=(",", ":")).encode()), + "group_count": len(heavy_result["groups"]), + "result_count": len(heavy_cards), + "requested_max_results": 100, + } + + fixture_server, fixture_thread, fixture_url = start_fetch_fixture() + try: + fetch_client = AsyncEggsearch(str(config)) + try: + fetch_durations = [] + fetch_peak_allocated = 0 + fetch_result = {} + for index in range(repeats): + fetch_started = time.perf_counter_ns() + tracemalloc.start() + fetch_result = await fetch_client.web_fetch( + url=f"{fixture_url}?sample={index}", max_chars=12000 + ) + _, peak_allocated = tracemalloc.get_traced_memory() + tracemalloc.stop() + fetch_peak_allocated = max(fetch_peak_allocated, peak_allocated) + fetch_durations.append((time.perf_counter_ns() - fetch_started) / 1_000_000) + if not fetch_result.get("fetched") or not fetch_result.get("text"): + raise RuntimeError(f"local web_fetch fixture failed: {fetch_result.get('errors')}") + document = fetch_result.get("document", {}) + results["fetch"] = { + **summarize(fetch_durations), + "serialized_bytes": len(json.dumps(fetch_result, separators=(",", ":")).encode()), + "text_chars": document.get("text_chars_returned", 0), + "requested_max_chars": 12000, + "tracemalloc_peak_bytes": fetch_peak_allocated, + "fixture_scope": "loopback-only", + } + finally: + await fetch_client.aclose() + finally: + fixture_server.shutdown() + fixture_server.server_close() + fixture_thread.join(timeout=2) + results["memory"]["after_fetch"] = memory_snapshot() + + cancellation_ms = [] + for round_index in range(repeats): + tasks = [ + asyncio.create_task(client.repo_search(query=f"cancel {round_index} {i}", include_local=True)) + for i in range(32) + ] + await asyncio.sleep(0) + started = time.perf_counter_ns() + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + cancellation_ms.append((time.perf_counter_ns() - started) / 1_000_000) + results["cancellation"]["cancel_32_and_drain_ms"] = summarize(cancellation_ms) + results["cancellation"]["post_cancel_recovery_ms"] = await timed_search(client, repeats) + close_started = time.perf_counter_ns() + await client.aclose() + results["client_close_ms"] = (time.perf_counter_ns() - close_started) / 1_000_000 + results["memory"]["after_close"] = memory_snapshot() + return results + + +def run_sequential_event_loop_benchmark(config, repeats): + durations = [] + for index in range(repeats): + started = time.perf_counter_ns() + asyncio.run(sequential_event_loop_sample(config, index)) + durations.append((time.perf_counter_ns() - started) / 1_000_000) + return summarize(durations) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--version", required=True) + parser.add_argument("--repeats", type=int, default=40) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--pip-report", type=Path, required=True) + parser.add_argument("--output", type=Path, default=Path("sbc-runtime.json")) + args = parser.parse_args() + installed = distribution("eggsearch") + if installed.version != args.version: + raise SystemExit(f"installed eggsearch {installed.version} does not match {args.version}") + wheel_sha256 = hashlib.sha256(args.wheel.read_bytes()).hexdigest() + report = json.loads(args.pip_report.read_text(encoding="utf-8")) + package_install = next( + ( + item + for item in report.get("install", []) + if item.get("metadata", {}).get("name", "").lower() == "eggsearch" + ), + None, + ) + if package_install is None or package_install.get("metadata", {}).get("version") != args.version: + raise SystemExit("pip report does not identify the requested eggsearch version") + download = package_install.get("download_info", {}) + report_sha256 = download.get("archive_info", {}).get("hashes", {}).get("sha256") + if not report_sha256 or report_sha256 != wheel_sha256: + raise SystemExit("downloaded wheel does not match the public pip install report digest") + if not download.get("url", "").startswith("https://files.pythonhosted.org/"): + raise SystemExit("pip report does not identify a public PyPI wheel URL") + with ZipFile(args.wheel) as archive: + native_path = next( + path + for path in archive.namelist() + if path.startswith("eggsearch/_native.") and path.endswith((".so", ".pyd")) + ) + installed_native = Path(installed.locate_file(native_path)) + if not installed_native.is_file(): + raise SystemExit("installed native extension is missing") + if hashlib.sha256(installed_native.read_bytes()).digest() != hashlib.sha256( + archive.read(native_path) + ).digest(): + raise SystemExit("installed native extension differs from the published wheel") + if not sys.platform.startswith("linux") or platform.machine() not in {"armv7l", "armv8l", "aarch64"}: + raise SystemExit("run this benchmark on a native Linux SBC installation") + if args.repeats < 3: + raise SystemExit("use at least three repeats") + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + workspace = root / "workspace" + workspace.mkdir() + for index in range(80): + text = " ".join([f"retry fixture evidence {index}"] * 160) + (workspace / f"fixture-{index:03}.md").write_text(text, encoding="utf-8") + config = local_config(root, workspace) + measured = asyncio.run(run_benchmarks(config, args.repeats)) + measured["sequential_asyncio_run_loop_ms"] = run_sequential_event_loop_benchmark( + config, args.repeats + ) + + libc_name, libc_version = platform.libc_ver() + available_tools = tool_names() + if len(available_tools) != 10 or len(set(available_tools)) != 10: + raise SystemExit("installed package did not expose ten unique tool names") + result = { + "package_version": installed.version, + "wheel": str(args.wheel), + "wheel_sha256": wheel_sha256, + "pypi_wheel_url": download["url"], + "pip_report_sha256": report_sha256, + "python": sys.version, + "machine": platform.machine(), + "platform": platform.platform(), + "libc": {"name": libc_name, "version": libc_version}, + "system": system_snapshot(installed), + "tool_count": len(available_tools), + "tool_names": available_tools, + "cold_import_ms": cold_import_ms(), + "wheel_bytes": args.wheel.stat().st_size, + "usage": {"user_cpu_seconds": getrusage(RUSAGE_SELF).ru_utime, "system_cpu_seconds": getrusage(RUSAGE_SELF).ru_stime}, + "measurements": measured, + } + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index ab37005..019f9d1 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,13 +1,14 @@ [build-system] -requires = ["maturin==1.9.5"] +requires = ["maturin==1.15.0"] build-backend = "maturin" [project] name = "eggsearch" version = "0.4.4" description = "Native asyncio bindings for the Eggsearch evidence engine" -requires-python = ">=3.11,<3.15" +requires-python = ">=3.11,<3.16" license = "MIT" +license-files = ["LICENSE-PYTHON"] readme = "PYTHON_README.md" classifiers = [ "Programming Language :: Python :: 3", @@ -15,6 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Rust", ] diff --git a/bindings/python/scripts/audit_armv7_wheel.py b/bindings/python/scripts/audit_armv7_wheel.py new file mode 100644 index 0000000..4da55f4 --- /dev/null +++ b/bindings/python/scripts/audit_armv7_wheel.py @@ -0,0 +1,73 @@ +import argparse +import re +import subprocess +import tempfile +import zipfile +from pathlib import Path + +from packaging.utils import parse_wheel_filename + + +def output(*command): + return subprocess.run(command, check=True, capture_output=True, text=True).stdout + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("wheel", type=Path) + args = parser.parse_args() + + name, _, _, tags = parse_wheel_filename(args.wheel.name) + if name != "eggsearch" or not tags: + raise SystemExit("unexpected wheel distribution") + if not all(tag.interpreter == "cp311" and tag.abi == "abi3" for tag in tags): + raise SystemExit("ARMv7 wheel must use the cp311 abi3 stable ABI") + platforms = {tag.platform for tag in tags} + if platforms != {"manylinux_2_28_armv7l"}: + raise SystemExit(f"unexpected ARMv7 wheel platform tags: {sorted(platforms)}") + + with zipfile.ZipFile(args.wheel) as archive: + native = [ + path + for path in archive.namelist() + if path.startswith("eggsearch/_native.") and path.endswith(".so") + ] + if len(native) != 1: + raise SystemExit("wheel must contain exactly one native shared object") + if any(Path(path).name in {"eggsearch", "eggsearch.exe"} for path in archive.namelist()): + raise SystemExit("wheel unexpectedly bundles the Eggsearch executable") + with tempfile.TemporaryDirectory() as directory: + binary = Path(directory) / "_native.so" + binary.write_bytes(archive.read(native[0])) + header = output("readelf", "-h", str(binary)) + attributes = output("readelf", "-A", str(binary)) + dynamic = output("readelf", "-d", str(binary)) + versions = output("readelf", "--version-info", str(binary)) + + if "ELF32" not in header or "Machine:" not in header or "ARM" not in header: + raise SystemExit("native extension is not a 32-bit ARM ELF") + if "Type:" not in header or "DYN" not in header or "Version5 EABI" not in header: + raise SystemExit("native extension is not a dynamically linked ARM EABI5 shared object") + if "Tag_ABI_VFP_args: VFP registers" not in attributes: + raise SystemExit("native extension does not declare the ARM hard-float calling convention") + if "libpython" in dynamic: + raise SystemExit("native extension unexpectedly links to libpython") + needed = sorted(set(re.findall(r"Shared library: \[(.*?)\]", dynamic))) + glibc = [tuple(map(int, value.split("."))) for value in re.findall(r"GLIBC_(\d+(?:\.\d+)+)", versions)] + if not glibc: + raise SystemExit("native extension has no inspectable GLIBC symbol requirements") + maximum = max(glibc) + if maximum > (2, 28): + raise SystemExit(f"native extension requires GLIBC {'.'.join(map(str, maximum))}, above 2.28") + + print(f"wheel={args.wheel.name}") + print("elf_class=ELF32") + print("machine=ARM") + print("abi=hard-float") + print("dynamic_dependencies=" + ",".join(needed)) + print(f"max_glibc={'.'.join(map(str, maximum))}") + print("tag=manylinux_2_28_armv7l") + + +if __name__ == "__main__": + main() diff --git a/bindings/python/scripts/smoke_installed_wheel.py b/bindings/python/scripts/smoke_installed_wheel.py new file mode 100644 index 0000000..d9cee70 --- /dev/null +++ b/bindings/python/scripts/smoke_installed_wheel.py @@ -0,0 +1,63 @@ +import asyncio +import tempfile +from importlib.metadata import distribution +from pathlib import Path + +from eggsearch import AsyncEggsearch, EggsearchError, tool_names + + +async def main(): + if len(tool_names()) != 10: + raise AssertionError("unexpected canonical tool inventory") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + workspace = root / "workspace" + workspace.mkdir() + (workspace / "README.md").write_text( + "Distinctive local ARM wheel fixture for bounded retry policy search.", + encoding="utf-8", + ) + config = root / "eggsearch.toml" + config.write_text( + "[search]\nmode = 'off'\ndefault_max_results = 10\n" + "max_results_cap = 50\nmax_query_chars = 512\ntimeout_ms = 8000\n" + "default_providers = ['duckduckgo']\nsanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\n\n" + "[fetch]\nenabled = false\ntimeout_ms = 8000\n" + "max_bytes = 2000000\nmax_chars_default = 12000\n" + "max_chars_cap = 50000\nredirect_limit = 5\n" + "allow_private_network = false\nallow_localhost = false\n" + "sanitize_output = true\n\n" + f"[local]\nenabled = true\nroots = ['{workspace.as_posix()}']\n", + encoding="utf-8", + ) + client = AsyncEggsearch(str(config)) + try: + try: + await client.execute("repo_search", {}) + except EggsearchError as error: + if error.code != "locator_invalid" or not error.message: + raise AssertionError("malformed input returned an unexpected error") from error + else: + raise AssertionError("malformed repository search input was accepted") + result = await client.repo_search(query="retry", include_local=True) + cards = [ + card + for group in result.get("groups", []) + for card in group.get("results", []) + ] + if not any(card.get("trust") == "local_trusted" for card in cards): + raise AssertionError("Mode-Off local search did not return trusted evidence") + finally: + await client.aclose() + try: + await client.provider_status(probe=False) + except EggsearchError as error: + if error.code != "client_closed": + raise AssertionError("closed client returned an unexpected error") from error + else: + raise AssertionError("closed client accepted a new operation") + print(distribution("eggsearch").version) + + +asyncio.run(main()) diff --git a/bindings/python/scripts/verify_release_artifacts.py b/bindings/python/scripts/verify_release_artifacts.py new file mode 100644 index 0000000..2b9f451 --- /dev/null +++ b/bindings/python/scripts/verify_release_artifacts.py @@ -0,0 +1,205 @@ +import argparse +import hashlib +import json +import tarfile +import zipfile +from email.parser import Parser +from pathlib import Path + +from packaging.specifiers import SpecifierSet +from packaging.utils import parse_sdist_filename, parse_wheel_filename + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("directory", type=Path) + parser.add_argument("version") + parser.add_argument("manifest", type=Path) + parser.add_argument("--source-sha", required=True) + parser.add_argument("--qualification-run-id", required=True) + args = parser.parse_args() + + wheels = sorted(args.directory.rglob("*.whl")) + sdists = sorted(args.directory.rglob("*.tar.gz")) + if len(wheels) != 6 or len(sdists) != 1: + raise SystemExit(f"expected six wheels and one sdist, found {len(wheels)} and {len(sdists)}") + + required_platforms = { + "x86_64": ("manylinux2014_x86_64", "manylinux_2_17_x86_64"), + "aarch64": ("manylinux2014_aarch64", "manylinux_2_17_aarch64"), + "macos_x86_64": ("macosx_10_12_x86_64",), + "macos_arm64": ("macosx_11_0_arm64", "macosx_12_0_arm64"), + "win_amd64": ("win_amd64",), + "armv7l": ("manylinux_2_28_armv7l",), + } + found = set() + records = [] + for wheel in wheels: + name, version, _, tags = parse_wheel_filename(wheel.name) + if name != "eggsearch" or str(version) != args.version: + raise SystemExit(f"unexpected distribution or version in {wheel.name}") + if not all(tag.interpreter == "cp311" and tag.abi == "abi3" for tag in tags): + raise SystemExit(f"unexpected interpreter ABI tags in {wheel.name}") + platforms = {tag.platform for tag in tags} + platform_id = next( + ( + key + for key, expected in required_platforms.items() + if any(platform.startswith(prefix) for platform in platforms for prefix in expected) + ), + None, + ) + if platform_id is None or platform_id in found: + raise SystemExit(f"unexpected or duplicate wheel platform in {wheel.name}: {platforms}") + found.add(platform_id) + with zipfile.ZipFile(wheel) as archive: + files = set(archive.namelist()) + package_files = { + path + for path in files + if path.startswith("eggsearch/") + } + dist_info_roots = { + path.split("/", 1)[0] + for path in files + if ".dist-info/" in path + } + if len(dist_info_roots) != 1: + raise SystemExit(f"unexpected wheel metadata roots in {wheel.name}") + native_files = { + path + for path in package_files + if path.startswith("eggsearch/_native.") and path.endswith((".so", ".pyd")) + } + required_package_files = { + "eggsearch/__init__.py", + "eggsearch/_native.pyi", + "eggsearch/py.typed", + "eggsearch/types.py", + } + if not required_package_files.issubset(package_files): + raise SystemExit(f"missing Python package files in {wheel.name}") + if len(native_files) != 1: + raise SystemExit(f"expected one native extension in {wheel.name}") + if package_files != required_package_files | native_files: + raise SystemExit(f"unexpected Python package contents in {wheel.name}") + if any(Path(path).name in {"eggsearch", "eggsearch.exe"} for path in files): + raise SystemExit(f"unexpected CLI executable in {wheel.name}") + dist_info_prefix = f"{next(iter(dist_info_roots))}/" + dist_info_files = {path for path in files if path.startswith(dist_info_prefix)} + required_dist_info = { + f"{dist_info_prefix}METADATA", + f"{dist_info_prefix}WHEEL", + f"{dist_info_prefix}RECORD", + f"{dist_info_prefix}licenses/LICENSE-PYTHON", + } + if not required_dist_info.issubset(dist_info_files): + raise SystemExit(f"missing license or wheel metadata in {wheel.name}") + allowed_dist_info = required_dist_info | { + f"{dist_info_prefix}sboms/eggsearch-python.cyclonedx.json" + } + if not dist_info_files.issubset(allowed_dist_info): + raise SystemExit(f"unexpected wheel metadata contents in {wheel.name}") + if files != package_files | dist_info_files: + raise SystemExit(f"unexpected top-level wheel contents in {wheel.name}") + metadata_path = f"{dist_info_prefix}METADATA" + metadata = Parser().parsestr(archive.read(metadata_path).decode()) + if metadata.get("Name") != "eggsearch" or metadata.get("Version") != args.version: + raise SystemExit(f"wheel metadata mismatch in {wheel.name}") + if SpecifierSet(metadata.get("Requires-Python", "")) != SpecifierSet(">=3.11,<3.16"): + raise SystemExit(f"unexpected Requires-Python in {wheel.name}") + if metadata.get("License-Expression") != "MIT": + raise SystemExit(f"unexpected wheel license in {wheel.name}") + if metadata.get("Description-Content-Type", "").split(";", 1)[0] != "text/markdown": + raise SystemExit(f"wheel is missing the Markdown project README in {wheel.name}") + if len(metadata.get_payload()) < 300: + raise SystemExit(f"wheel project README is unexpectedly short in {wheel.name}") + records.append({"file": wheel.name, "sha256": hashlib.sha256(wheel.read_bytes()).hexdigest()}) + + if found != set(required_platforms): + raise SystemExit(f"wheel platform inventory mismatch: {found}") + audits = sorted(args.directory.rglob("armv7-audit.txt")) + if len(audits) != 1: + raise SystemExit(f"expected one ARMv7 ELF audit record, found {len(audits)}") + audit = audits[0].read_text(encoding="utf-8") + required_audit_lines = { + "elf_class=ELF32", + "machine=ARM", + "abi=hard-float", + "tag=manylinux_2_28_armv7l", + } + if not required_audit_lines.issubset(set(audit.splitlines())): + raise SystemExit("ARMv7 ELF audit record is incomplete") + glibc_line = next((line for line in audit.splitlines() if line.startswith("max_glibc=")), None) + if glibc_line is None: + raise SystemExit("ARMv7 ELF audit record is missing its glibc floor") + if not any(line.startswith("dynamic_dependencies=") for line in audit.splitlines()): + raise SystemExit("ARMv7 ELF audit record is missing dynamic dependencies") + glibc = tuple(map(int, glibc_line.split("=", 1)[1].split("."))) + if glibc > (2, 28): + raise SystemExit(f"ARMv7 wheel exceeds its manylinux_2_28 floor: {glibc}") + audit_record = { + "file": str(audits[0].relative_to(args.directory)), + "sha256": hashlib.sha256(audits[0].read_bytes()).hexdigest(), + } + sdist_name, sdist_version = parse_sdist_filename(sdists[0].name) + if sdist_name != "eggsearch" or str(sdist_version) != args.version: + raise SystemExit(f"unexpected source distribution: {sdists[0].name}") + with tarfile.open(sdists[0], "r:gz") as archive: + members = archive.getmembers() + names = {member.name for member in members} + roots = {name.split("/", 1)[0] for name in names} + if len(roots) != 1: + raise SystemExit("sdist contains files outside its single distribution root") + for member in members: + path = Path(member.name) + if path.is_absolute() or ".." in path.parts: + raise SystemExit(f"unsafe path in sdist: {member.name}") + if path.suffix in {".so", ".pyd", ".pyc", ".dll", ".dylib"}: + raise SystemExit(f"unexpected compiled artifact in sdist: {member.name}") + if path.suffix.lower() in {".log", ".ansi", ".cast", ".dump", ".pem", ".key", ".p12"}: + raise SystemExit(f"unexpected transcript, dump, or credential in sdist: {member.name}") + if member.isfile() and member.mode & 0o111: + raise SystemExit(f"unexpected executable file in sdist: {member.name}") + if any(part in {"target", ".git", "__pycache__"} for part in path.parts): + raise SystemExit(f"unexpected generated directory in sdist: {member.name}") + if path.name.lower() in {".env", "id_rsa", "id_ed25519", "credentials.json"} or path.name.lower().startswith(".env."): + raise SystemExit(f"unexpected credential-like file in sdist: {member.name}") + if not any(name.endswith("/Cargo.lock") and "bindings/python" not in name for name in names): + raise SystemExit("sdist is missing the root Cargo.lock") + if not any(name.endswith("/bindings/python/Cargo.lock") for name in names): + raise SystemExit("sdist is missing bindings/python/Cargo.lock") + if not any(name.endswith("/LICENSE") for name in names): + raise SystemExit("sdist is missing its project license") + if not any(name.endswith("/README.md") for name in names): + raise SystemExit("sdist is missing the root project README") + if not any(name.endswith("/bindings/python/PYTHON_README.md") for name in names): + raise SystemExit("sdist is missing the Python package README") + metadata_member = next((member for member in members if member.name.endswith("/PKG-INFO")), None) + if metadata_member is None: + raise SystemExit("sdist is missing PKG-INFO") + metadata = Parser().parsestr(archive.extractfile(metadata_member).read().decode()) + if metadata.get("Name") != "eggsearch" or metadata.get("Version") != args.version: + raise SystemExit("sdist metadata mismatch") + if SpecifierSet(metadata.get("Requires-Python", "")) != SpecifierSet(">=3.11,<3.16"): + raise SystemExit("sdist has unexpected Requires-Python metadata") + if metadata.get("License-Expression") != "MIT" or len(metadata.get_payload()) < 300: + raise SystemExit("sdist is missing its expected license or project README") + records.append({"file": sdists[0].name, "sha256": hashlib.sha256(sdists[0].read_bytes()).hexdigest()}) + args.manifest.write_text( + json.dumps( + { + "version": args.version, + "source_sha": args.source_sha, + "qualification_run_id": args.qualification_run_id, + "artifacts": records, + "armv7_audit": audit_record, + }, + indent=2, + ) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/docs/python-library.md b/docs/python-library.md index bbe4ecb..779f32a 100644 --- a/docs/python-library.md +++ b/docs/python-library.md @@ -7,13 +7,13 @@ server, or listener. The package is an independent PyO3 extension under ## Build and use locally -Use Rust 1.89, CPython 3.11–3.14, and a virtual environment. From the +Use Rust 1.89, CPython 3.11–3.15, and a virtual environment. From the repository root: ```bash python -m venv .venv source .venv/bin/activate -python -m pip install -e 'bindings/python[dev]' maturin==1.9.5 +python -m pip install -e 'bindings/python[dev]' maturin==1.15.0 maturin develop --manifest-path bindings/python/Cargo.toml --locked python -m pytest -q bindings/python/tests pyright bindings/python/python/eggsearch bindings/python/examples @@ -121,15 +121,32 @@ remote source text uses `external_untrusted`. Preserve the `trust` and ## Compatibility and features -The binding enables PyO3's `abi3-py311` stable ABI, targeting CPython 3.11 and -newer. M004 qualifies the same platform wheel on CPython 3.11, 3.12, 3.13, and -3.14. Linux wheels use the manylinux2014 baseline; native targets are Linux -x86_64 and aarch64, macOS x86_64 and arm64, and Windows x86_64. CI builds on -each target and installs each exact wheel on all four interpreters without -installing Rust in the consumer job. This support statement is conditional on -the corresponding M004 workflow jobs passing for the released candidate. +The binding enables PyO3's `abi3-py311` stable ABI, targeting GIL-enabled +CPython 3.11 and newer. The active PyPI distribution qualification targets +CPython 3.11 through 3.15 on each supported platform; only candidates that pass +the exact 25-cell workflow matrix qualify. Linux wheels use the manylinux2014 +baseline; native targets are Linux x86_64 and aarch64, macOS x86_64 and arm64, +and Windows x86_64. CI builds on each target and installs each exact wheel on +all five interpreters without installing Rust in the consumer job. These +claims remain conditional on the corresponding exact-candidate jobs passing. + +PyPy, free-threaded CPython, and Windows ARM64 are not claimed. ARMv7 is +planned as experimental SBC support only after its separate build and emulated +installed-wheel qualification passes. The release candidate uses a pinned +manylinux cross toolchain and targets the `manylinux_2_28_armv7l` compatibility +tag; its glibc floor and hard-float ABI must pass artifact audit and emulated +installation before publication. That support is limited to ARMv7 hard-float +Linux with glibc 2.28 or newer. ARMv6, soft-float, Windows ARM64, and Alpine or +other musl systems are not included. After the first release appears on PyPI, +SBC users must require a wheel so pip fails on an unsupported target instead +of trying to compile from source: -PyPy, free-threaded CPython, ARMv7, and Windows ARM64 are not claimed. The +```bash +python -m pip install --only-binary=:all: eggsearch +``` + +Do not build the source distribution on an SBC. The source distribution is a +developer fallback for environments outside the published wheel matrix. The extension package enables no optional Rust `browser`, `egress`, or `pdf` features. The native library uses the same Eggsearch configuration, provider credential environment variables, bounded operations, sanitization, and trust @@ -140,10 +157,36 @@ An sdist build requires Rust 1.89, a C toolchain, and the Cargo dependencies in `bindings/python/Cargo.lock`. It can build offline when those locked crate sources are already in Cargo's local cache; a fresh build must be able to fetch them from Cargo registries. Native wheels avoid those build requirements at -install and runtime. After a separately authorized publication, -upgrade with `python -m pip install --upgrade eggsearch` and remove with +install and runtime. After the M003 first publication is verified, upgrade +with `python -m pip install --upgrade eggsearch` and remove with `python -m pip uninstall eggsearch`. +## Post-publication SBC benchmark + +M004 runs only on native ARMv7 or aarch64 SBC hardware after publication. Install +the exact public wheel with `--only-binary=:all:`, then run the standard-library +harness outside the repository checkout: + +```bash +python -m pip install --only-binary=:all: --report install-report.json eggsearch==VERSION +python -m pip download --only-binary=:all: --no-deps --dest wheelhouse eggsearch==VERSION +python /path/to/eggsearch/bindings/python/scripts/smoke_installed_wheel.py +python /path/to/eggsearch/bindings/python/benchmarks/sbc_runtime.py \ + --version VERSION --wheel /path/to/wheelhouse/eggsearch-*.whl \ + --pip-report install-report.json +``` + +The harness verifies that the wheel matches the public PyPI URL and SHA-256 in +the pip install report, including the installed native extension. It uses a +local Mode-Off search fixture and a loopback-only fetch fixture. It records cold +import, first operation, repeated sequential event loops, search latency at +1–32 concurrent calls, fetch latency and output near configured character +bounds, cancellation and close, memory, thread and descriptor counts, wheel +size, OS, CPU, RAM, libc, storage, and available thermal or throttling +information. It performs no wheel build and calls no remote providers. Keep +the JSON output with the exact published wheel hash and hardware inventory for +the M004 closure record. + See [Tool Matrix](tool-matrix.md), [Agent Workflows](agent-workflows.md), and [Python Library Architecture](../architecture/python-library.md) for tool semantics, result trust, and implementation ownership. diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 6bbbcfb..df8e723 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -27,3 +27,12 @@ Short operational checklist. The authoritative release process lives in - [ ] Confirm the Eggpack-generated `Eggpack candidate builds` workflow creates a complete draft release - [ ] Review binary checksums and publish the draft release manually - [ ] Verify the published release has exactly 19 assets (7 executables, 7 `.sha256` sidecars, `install.sh`, `install.ps1`, `release-manifest.json`, `install-exact.sh`, `install-exact.ps1`) and run external Unix/Windows installer smoke + +## Separate Python package publication + +- [ ] Confirm M001 and M002 exact-SHA qualification and root CI are green +- [ ] Configure PyPI Trusted Publishing for `eggstack/eggsearch`, workflow `release-python-pypi.yml`, environment `pypi` +- [ ] Create GitHub environment `pypi` before dispatching +- [ ] Dispatch the Python publication workflow with the qualified SHA, Python qualification run ID, and synchronized version +- [ ] Confirm public binary-only installs and the M003 closure record before announcing availability +- [ ] Schedule M004 benchmarks on native SBC devices after publication diff --git a/docs/release.md b/docs/release.md index 4775dfc..1137099 100644 --- a/docs/release.md +++ b/docs/release.md @@ -57,6 +57,30 @@ same commit. Do not dispatch the release producer until the matching crate version is visible on crates.io. See `architecture/packaging.md` for the live producer contract. +## Python package publication + +The native Python distribution has its own release path and does not use root +Rust tags, crates.io publication, Eggpack, or the 19-asset binary inventory. +The controlling workstream is `plans/subsystems/python-pypi-distribution-roadmap.md`. +Its first publication must use the exact candidate SHA whose five-platform, +25-cell interpreter matrix, ARMv7 emulated install, source rebuild, audits, and +root CI all passed. + +Before dispatching `.github/workflows/release-python-pypi.yml`, configure the +PyPI Trusted Publisher for owner `eggstack`, repository `eggsearch`, workflow +`release-python-pypi.yml`, and pre-create the GitHub environment `pypi`. The +publisher workflow requires the exact candidate SHA, successful Python +qualification run ID, and synchronized package version. Its preflight verifies +the environment exists, then verifies the source and six-wheel plus sdist +inventory before the OIDC publish job. Dispatching the workflow is the explicit +publication action. The publication is complete only after independent public +binary-only installs pass on Linux x86_64/aarch64, macOS x86_64/arm64, Windows +x86_64, and emulated ARMv7. + +Do not announce PyPI availability or update the public installation claim +until that post-publication verification and M003 closure record are complete. +Physical SBC measurements belong to M004 and happen after the initial release. + ## Publication and binary release ```bash diff --git a/plans/closure/python-pypi-distribution/001-status.md b/plans/closure/python-pypi-distribution/001-status.md new file mode 100644 index 0000000..718e235 --- /dev/null +++ b/plans/closure/python-pypi-distribution/001-status.md @@ -0,0 +1,72 @@ +# M001 — Python 3.15, PyO3 Toolchain and Stable-ABI Qualification Closure + +Status: closed +Candidate: `1135608af5d625cd72db6704adf3bcabd1a58ab1` +Implementation plan: `plans/implementation/python-pypi-distribution/001-python315-toolchain-and-abi3.md` +Roadmap: `plans/subsystems/python-pypi-distribution-roadmap.md` M001 +Pull request: #10 + +## Candidate lineage + +The candidate is the exact source SHA used for every qualifying check and every +artifact below. The work was implemented in `3c531920`, corrected for clean +hosted metadata inspection in `08769df`, and finalized in `1135608` with a +packaged Python license file and verifier assertion. No distributable source +changed after the qualifying SHA. + +## Result + +CPython 3.11–3.15 is supported through `cp311-abi3`. PyO3 0.30.1 and +`pyo3-async-runtimes` 0.30.0 build on Rust 1.89; Maturin is pinned to 1.15.0. +The package metadata advertises `>=3.11,<3.16`, includes the Python 3.15 +classifier, and does not claim PyPy, free-threaded Python, or Python 3.10. +The five existing native platform families each have one ABI3 wheel. All five +interpreters installed and exercised the matching wheels in clean consumer +environments. + +The binding package now declares `LICENSE-PYTHON`; its bytes match the root +MIT license. Maturin places it under `.dist-info/licenses/` and records it in +wheel metadata. The unique filename avoids an sdist path collision with the +root `LICENSE`. + +## Exact-SHA verification + +| Check | Result | +|---|---| +| Python native qualification, run `40729011504` | **34/34 jobs passed**: six platform wheel builds including ARMv7, dependency audit, extracted-sdist offline rebuild, ARMv7 emulated install, and all 25 CPython 3.11–3.15 clean installed-wheel smokes across the five native platform families. | +| Root CI, run `40729011529` | `ci` and Windows portability passed; canonical `make check` gate green. | +| Release drift guard, run `40729011257` | Eggpack workflow drift, release contract and pinned action checks passed. | +| Egress qualification, run `40729042102` | All target checks passed, including macOS ARM64 and Rust 1.89 all-features. | +| Artifact verifier | Passed against the complete hosted artifact bundle with exact source SHA and qualification run ID. It accepted exactly six wheels, one sdist, and one ARMv7 audit record. | +| `cargo +1.89.0 publish --dry-run --locked` | Passed on the clean candidate. The existing crates.io 0.4.4 warning is expected; no upload occurred. Cargo packaged 290 files. | +| Binding dependency audit | Both Cargo graphs passed `cargo deny`; pinned Python `pip-audit --local` reported no known vulnerabilities. | +| Local package checks | Rust 1.89 binding check, local Maturin wheel build and sdist generation passed; the generated wheel contained `LICENSE-PYTHON` and the sdist preserved both root and Python license paths. | + +The root Cargo dependency graph and Eggpack asset contract are unchanged by the +Python binding work. No new Python runtime dependency was added. + +## Hosted release artifact digests + +These SHA-256 values are from qualification run `40729011504`. The verifier +manifest was generated from the downloaded artifacts at +`/tmp/eggsearch-qualification-1135608/release-manifest.json`. + +| Artifact | SHA-256 | +|---|---| +| `eggsearch-0.4.4-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl` | `96247b088c2ae924e49c459c5a29b7b9ac348c776a9c928e1eaafc09f032a4e0` | +| `eggsearch-0.4.4-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl` | `840e8689a9ec52d26e480f3946e8a389ad0bdab816940681a1431b2faea52ccb` | +| `eggsearch-0.4.4-cp311-abi3-macosx_10_12_x86_64.whl` | `cb5dd563ee71f1aa8844c4c454effad17c1838ce315c01f14776dbf2733db7ad` | +| `eggsearch-0.4.4-cp311-abi3-macosx_11_0_arm64.whl` | `adaadee8bd530f302b642a48886ca723f19c245c5249e107aa885a0cfe78349e` | +| `eggsearch-0.4.4-cp311-abi3-win_amd64.whl` | `62bedfea95581d906e5f4749310c2013a277754e05f177d93d4f08a271bc8340` | +| `eggsearch-0.4.4-cp311-abi3-manylinux_2_28_armv7l.whl` | `530c31f51018886caaa70659e037ed6ceff5a4d053f762ebef46f92c8ec82eb0` | +| `eggsearch-0.4.4.tar.gz` | `42e57e1c9a4ec02b97d715cc28f522e7effde2e1cb9d0ca43ee0fecaac7ae436` | + +ARMv7 audit record SHA-256: +`7d574ce6c34f7874666b8525cca70ab145bd5d9a4bc0db2533f7f58ea34f5f76`. + +## Scope and handoff + +M001 is closed. No publication was performed. M003 remains the controlled +publication gate and consumes candidate `1135608`; it requires a PyPI Trusted +Publisher plus the GitHub `pypi` environment approval configuration. M002's +ARMv7 evidence is separately recorded in `002-status.md`. diff --git a/plans/closure/python-pypi-distribution/002-status.md b/plans/closure/python-pypi-distribution/002-status.md new file mode 100644 index 0000000..7179580 --- /dev/null +++ b/plans/closure/python-pypi-distribution/002-status.md @@ -0,0 +1,46 @@ +# M002 — Hosted ARMv7 and Linux Wheel Expansion Closure + +Status: closed +Candidate: `1135608af5d625cd72db6704adf3bcabd1a58ab1` +Implementation plan: `plans/implementation/python-pypi-distribution/002-hosted-armv7-and-linux-wheel-expansion.md` +Roadmap: `plans/subsystems/python-pypi-distribution-roadmap.md` M002 +Qualification run: `40729011504` + +## Mandatory ARMv7 artifact + +The hosted cross-build produced +`eggsearch-0.4.4-cp311-abi3-manylinux_2_28_armv7l.whl` with SHA-256 +`530c31f51018886caaa70659e037ed6ceff5a4d053f762ebef46f92c8ec82eb0`. +The pinned `readelf` audit reported: + +- ELF32, ARM EABI hard-float, `manylinux_2_28_armv7l` tag. +- Maximum GLIBC symbol version 2.28. +- Dynamic dependencies: `ld-linux-armhf.so.3`, `libc.so.6`, `libdl.so.2`, + `libgcc_s.so.1`, `libm.so.6`, `libpthread.so.0`, and `librt.so.1`. +- No bundled Eggsearch executable; the wheel contains the ABI3 extension, + Python package and stubs, metadata, license and SBOM. + +The exact hosted wheel was installed binary-only under QEMU in the pinned +PyPA manylinux 2.31 ARMv7 image on CPython 3.11.18 and 3.15.2. Each run passed +`23` tests with `6` expected skips, plus the installed-wheel ten-tool smoke. +Both reported `platform.machine() == "armv7l"` and Eggsearch 0.4.4. The pytest +cache emitted one warning because the checked-out source was mounted read-only; +the test results were successful. + +## Linux coverage and artifact inventory + +The existing manylinux 2.17 x86_64 and aarch64 wheels, macOS x86_64 and arm64 +wheels, and Windows x86_64 wheel all built and passed the M001 25-job +CPython/interpreter matrix. The release artifact verifier passed on the exact +six-wheel-plus-sdist bundle. All artifact hashes are recorded in +`001-status.md`. + +Optional `musllinux_1_2` x86_64/aarch64 and conditional ARMv7/i686 wheels are +deferred; this release makes no musl or i686 compatibility claim. + +## Limits and handoff + +No physical SBC was compiled on or benchmarked. ARMv7 support is ABI- and +emulation-qualified, while hardware-model, memory and performance uncertainty +remains experimental. M004 begins only after first PyPI publication and access +to native ARMv7/aarch64 devices. M002 is closed with the hosted evidence above. diff --git a/plans/implementation/python-pypi-distribution/001-python315-toolchain-and-abi3.md b/plans/implementation/python-pypi-distribution/001-python315-toolchain-and-abi3.md new file mode 100644 index 0000000..7951b39 --- /dev/null +++ b/plans/implementation/python-pypi-distribution/001-python315-toolchain-and-abi3.md @@ -0,0 +1,37 @@ +# M001 — Python 3.15, PyO3 Toolchain and Stable-ABI Qualification + +Status: closed +Primary class: infrastructure +Source: `plans/subsystems/python-pypi-distribution-roadmap.md` M001. +Execution candidate: branch `plan/python-pypi-prebuilt-arm-release-20261028`, SHA `1135608af5d625cd72db6704adf3bcabd1a58ab1`, Eggsearch 0.4.4. +Hard dependency: closed Python native M008 (`plans/closure/python-native-library/008-status.md`); interface guarantee: `abi3-py311` and ten-tool SDK contract. +Authority: accepted ADR-0005; long-term specification §§6–9 and `plans/003-planning-process.md`. + +## Objective and current evidence + +Qualify normal CPython 3.11–3.15 on the current five platform families and remove version-bound packaging defects **before** a controlled first PyPI release. Retain one stable-ABI wheel per platform rather than building a wheel for each interpreter. Python 3.15 is final as of 2026-10-09; not experimental. Current `bindings/python/pyproject.toml` explicitly excludes it (`>=3.11,<3.15`); `bindings/python/Cargo.toml` pins PyO3 0.29.3 and `pyo3-async-runtimes` 0.29.0; Maturin is 1.9.5. Workflow tests only 3.11–3.14 and hardcodes `'0.4.4'` in sdist smoke. M008's green 0.4.3 matrix is historical evidence only. + +Upstream candidate versions verified as available on 2026-10-28: PyO3 0.30.1, `pyo3-async-runtimes` 0.30.0, Maturin 1.15.0. These are **starting candidates, not forced upgrades**: check current lockfile resolution, Rust 1.89 compatibility and `future_into_py` cancellation behavior before pinning, and select the smallest coherent set proven by tests. + +Execution selected that coherent version set. The binding compiles with Rust 1.89.0; the release wheel suite passes 23 tests with six expected debug-gate skips; local `maturin develop` passes 28 tests with one skip, including in-flight cancellation and close. Extracted sdist rebuild from the sdist root succeeds with Cargo offline. Both Cargo dependency graphs pass `cargo deny` after updating `webpki-roots` to 1.0.10, `rustls-webpki` to 0.103.16, and Tokio to 1.53.3; the pinned Python build/audit environment reports no known vulnerabilities. Hosted qualification run `40729011504` passed all 25 CPython 3.11–3.15 smoke cells on the five native platform families at the exact candidate SHA. The complete M001 evidence is recorded in `plans/closure/python-pypi-distribution/001-status.md`. + +## Non-goals / preserved contracts + +No Python 3.10, PyPy, free-threaded/abi3t claim, Rust API general stabilization, new tool semantics, subprocess wrapper, on-SBC builds, Eggpack edits or PyPI upload in M001. Root dependency graph, ten-tool request/response semantics, no-implicit-config API, side-effect-free import, bounded concurrency, cancellation and trust policy must not regress. Treat C-extension-only changes as isolated to the binding package. + +## Ordered work packages + +1. Inspect current manifests, both `Cargo.lock` files, CI candidate and PyO3 3.15 compatibility. Upgrade matching PyO3 and async-runtime pair if required; update Maturin/PEP 517 build requirement and pinned action version where compatible. Run `cargo tree` for both locks; preserve Rust 1.89 in the binding. Do not use a Python-version bypass variable to conceal unsupported ABI behavior. +2. Set `requires-python = ">=3.11,<3.16"`, add Python 3.15 classifier, update `docs/python-library.md` and `bindings/python/PYTHON_README.md` to describe normal `abi3-py311` support and explicitly exclude free-threaded Python. Ensure the wheel metadata and generated tags reflect only tested interpreters. Keep the distribution and import name `eggsearch`. +3. Extend `.github/workflows/python-native-qualification.yml` with CPython 3.15 clean installed-wheel smoke for Linux glibc x86_64/aarch64, macOS x86_64/arm64 and Windows x86_64; preserve the five-wheel ABI3 build approach. If hosted `setup-python` lacks a 3.15 interpreter, install a trusted final CPython build in the consumer job and document provenance. Do not substitute build-only checks for interpreter use. +4. Remove the hardcoded version `'0.4.4'` from sdist smoke and use synchronized `Cargo.toml`/`pyproject.toml` metadata; add guard for all three versions, supported Python metadata, and wheel name/abi3 tags. Preserve offline rebuilt sdist from extraction and complete clean-venv consumer testing. Update pinned pytest/tooling where security scanning requires it, with explicit repeat qualification after any toolchain change. +5. Re-run deterministic binding tests, schema/TypedDict coverage, 10-operation fixture calls, Mode-Off zero remote dispatch, import-no-side-effects, cancellation including in-flight release/close, sequential event loops, and Python 3.15 (including import-related changes). Include debug-only bridge tests via local `maturin develop`, and separate release-wheel smoke (debug gates absent). +6. Run dependency/license/vulnerability audits separately for both Cargo graphs and the Python audit/build environment. Record any outstanding finding, severity, owner, and whether it affects published bytes. Keep the workflow actions SHA-pinned and protect root package boundaries. + +## Concrete verification + +From repository root where applicable: `make check`, `make packaging-check`, `cargo publish --dry-run --locked` on a clean tree, `cargo tree --locked --manifest-path bindings/python/Cargo.toml`, `cargo deny check` inside `bindings/python`, `maturin build --manifest-path bindings/python/Cargo.toml --release --sdist --locked`, `python -m pytest -q bindings/python/tests`, `pyright bindings/python/python/eggsearch bindings/python/examples`. Drive `Python native qualification` on an exact pushed commit; download each platform wheel into separate clean consumer environments with no Rust/CC/Cargo and install using `python -m pip install --no-index --no-deps --only-binary=:all: `. Test **each** CPython 3.11–3.15 on every existing wheel platform, not simply 3.15 on x86_64. Validate metadata via `importlib.metadata` and `packaging.tags`. + +## Acceptance, stop and closure + +Accepted only if all 25 interpreter/platform smoke cells pass on the **same candidate**, sdist extraction/rebuild is green, Python 3.15 is explicitly supported in wheel metadata and actually imports, no code/tool behavior regressions, full security audits are green or explicitly blocked by actionable findings, and root packaging remains unchanged. If upstream ABI dependencies cannot support 3.15 with Rust 1.89, block and record the dependency and the smallest correction instead of falsely advertising compatibility. The closure file `plans/closure/python-pypi-distribution/001-status.md` must include exact SHA, versions, wheel digests, 25-job matrix, commands/outcomes, security and compatibility notes, docs, and residuals. Register closure with roadmap/registry in the same commit. No PyPI upload in this milestone. diff --git a/plans/implementation/python-pypi-distribution/002-hosted-armv7-and-linux-wheel-expansion.md b/plans/implementation/python-pypi-distribution/002-hosted-armv7-and-linux-wheel-expansion.md new file mode 100644 index 0000000..7d249e3 --- /dev/null +++ b/plans/implementation/python-pypi-distribution/002-hosted-armv7-and-linux-wheel-expansion.md @@ -0,0 +1,49 @@ +# M002 — Hosted ARMv7 and Broad Linux Prebuilt Wheels, Experimental SBC Support + +Status: closed +Primary class: infrastructure +Source: `plans/subsystems/python-pypi-distribution-roadmap.md` M002. +Execution candidate: branch `plan/python-pypi-prebuilt-arm-release-20261028`, SHA `1135608af5d625cd72db6704adf3bcabd1a58ab1`, root/binding 0.4.4. +Interface dependency: M001 preserves CPython `abi3-py311` and the Rust 1.89 SDK; implement build support in parallel, then integrate exact candidate for M003. Operational dependency: first PyPI release needs passing required wheel qualifications. +Authority: ADR-0005, `plans/003-planning-process.md`, Python M004/M008 historical closure, and `architecture/python-library.md`. + +## Objective and baseline + +Produce an **installable precompiled ARMv7 hard-float wheel** for SBC-class 32-bit Linux without ever compiling the package on a Raspberry Pi, Le Potato, or other resource-constrained board. Expand Linux platform coverage with additional prebuilt wheels when qualified. Document SBC support as **experimental** until post-publication physical-device benchmarking. Existing Python workflow builds `manylinux2014` x86_64/aarch64, macOS Intel/ARM64 and Windows x86_64, all stable ABI; no ARMv7/musllinux wheels today. + +Execution selects `ghcr.io/rust-cross/manylinux_2_28-cross@sha256:b1f069c62f7744faa6db5f1ab28b11d59f72e161d8f03f6b0a388fe1cecc065d` and requires the generated wheel to retain its `manylinux_2_28_armv7l` tag after Maturin's audit. This is a broader glibc compatibility floor than the original 2.31 candidate. The independent install smoke runs in `quay.io/pypa/manylinux_2_31_armv7l@sha256:ff4a72812594311c071eebce43efcaa9061bd1205649d864c1054d4e56e6bdae` under QEMU. The pinned runtime image config declares Linux ARM. If either image or tag behavior differs, stop and update the decision with observed evidence before publication. + +The workflow has a separate ARMv7 cross-build artifact, a `readelf` audit of ELF class, EABI, hard-float attributes, dynamic dependencies and maximum GLIBC symbol version, and QEMU installed-wheel jobs under CPython 3.11 and 3.15. The release verifier requires the six mandatory platform families and checks that the audit record accompanies the exact qualification artifacts. The image digests were checked against their registries on 2026-10-28. Hosted build/audit/emulated install evidence passed on exact candidate `1135608af5d625cd72db6704adf3bcabd1a58ab1`, run `40729011504`; no physical SBC build or benchmark has been performed. Closure evidence is recorded in `plans/closure/python-pypi-distribution/002-status.md`. + +Upstream PyPA `manylinux_2_31_armv7l` image (`quay.io/pypa/manylinux_2_31_armv7l`) supports a Linux glibc 2.31 baseline and the `armv7l` platform tag. PyPA `musllinux_1_2` images exist for x86_64/aarch64/armv7l. These are *build infrastructure candidates*, not proof that the project or its Rust dependencies link on ARMv7. Choose an audited and reproducible CI recipe (hosted QEMU/binfmt running the proper image, or true cross-compile plus QEMU consumer smoke), with all command details and image digests recorded. + +## Mandatory and optional targets + +Mandatory for first publication: keep the five existing wheel families and add `armv7-unknown-linux-gnueabihf` producing a Python `cp311-abi3-manylinux_2_28_armv7l`-compatible wheel, with exact tag determined by audit. Its floor is glibc 2.28, **not** manylinux2014 and not all 32-bit Linux; does not include ARMv6, soft-float, Android, or Alpine/musl. The independent install test runs under the official manylinux 2.31 ARMv7 userland. Existing Linux aarch64 covers 64-bit SBC systems only if distribution/libc baseline matches. + +Preferred additional builds: `musllinux_1_2_x86_64`, `musllinux_1_2_aarch64`. Optional/conditionally deferred: `musllinux_1_2_armv7l`, `manylinux2014_i686`. Record explicit decision/evidence for any supplemental platform deferred so the release documentation names only actual wheels; inability to add optional targets must not delay a green mandatory release. + +Execution decision: defer both preferred musllinux wheels from the first inventory. The release verifier and consumer workflow are scoped to the six mandatory platform families; no musllinux compatibility is claimed. This keeps the first publication focused on the required CPython ABI matrix and ARMv7 hard-float gate. Revisit musllinux in a separately qualified expansion after M003 if hosted builds and matching consumer environments are available. + +## Non-goals / invariants + +**No compilation, Cargo, C linker, Rust installation, wheel rebuild, or `pip install` source fallback on any physical SBC before release.** Physical SBC operation and memory/performance experiments are M004 **after publication**. Do not add new runtime dependencies or change the root Eggpack ARMv7 executable pipeline. Preserve `abi3-py311`, bounded tool semantics, credentials/config and security; never turn off manylinux audit, retag a `linux_*` wheel as manylinux, or label a 64-bit wheel as 32-bit ARM. + +## Ordered work packages + +1. Derive an explicit platform grid from PyPI compatibility tags: `cp311-abi3`, `armv7l`, manylinux glibc floor, musllinux versus glibc, Python 3.11–3.15. Inspect `eggfetch-core`/TLS/crypto dependency target support and CPU instructions, Rust 1.89 ARMv7 hard-float toolchain/linker availability, and the upstream images' architecture. Document minimum supported OS/toolchain, avoid assumptions about Pi model. +2. Implement the ARMv7 job in `.github/workflows/python-native-qualification.yml` with hosted CI (e.g., x86_64 host + QEMU static/binfmt + pinned manylinux ARMv7 image, native ARM64 if it can reliably execute AArch32, or auditable cross compiler). Pin actions and container digests/versions. Build *once* with `--release --locked`; inspect Maturin audit/tag result, ELF class, ARM EABI hard-float, dynamic deps, required GLIBC symbols, correct `_native.abi3.so`, importable Python package and no CLI executable in wheel. A cross-compile alone is insufficient. +3. Download the **exact generated wheel** into an ARMv7 userland under QEMU and perform a binary-only install without Cargo or compiler. Verify `platform.machine()`, `sys.version_info`, `importlib.metadata`, `eggsearch.tool_names()`, local Mode-Off `repo_search`/source evidence, representative ten-operation semantics/errors, bounded cancellation/close and no import network/process side effects. Test at minimum CPython 3.11 and 3.15; when practical use 3.12–3.14 also, because the stable ABI is otherwise shared. Log emulator and distro details so this evidence is not mistaken for physical Pi qualification. +4. Add `musllinux_1_2` x86_64/aarch64 jobs with comparable hosted build/audit/import tests, and ARMv7 musl/i686 only if time/toolchain permits. Use a musl userland for musl smokes, not a glibc container. Verify absence of forbidden glibc deps and ensure compatibility tags match published platform. Do not silently substitute one libc family for another. +5. Add an artifact verifier that rejects wrong arch/libc/ABI tags, missing mandatory wheel, duplicate wheel platform tags, sdist contamination, root executable bundling, and wheel whose contents are not installable; verify `--only-binary=:all:` from exact candidate wheelhouse. Add deterministic Python/Rust guard tests if practical. +6. Update `docs/python-library.md`, `bindings/python/PYTHON_README.md`, root README's short Python link and support matrix only as appropriate. Clearly label ARM/SBC support experimental. State example SBC install `python -m pip install --only-binary=:all: eggsearch`; explain pip platform checks and that an incompatible host should **fail without attempting a local source build**. State explicitly that `pip install eggsearch` without `--only-binary` may attempt an sdist build on unsupported platforms, which is **not recommended on SBCs**. Disclose glibc/musl/ARMv7 vs ARM64 distinctions and post-release test plan. + +## Verification and release boundary + +Run `make check`, `make packaging-check`, binding sdist and audit checks, plus workflow ARMv7 compile-and-ELF audit, **emulated installed-wheel smoke** and architecture/libc tag assertions. Reuse existing 5-platform × 3.11–3.15 qualification from M001 and pin the exact tested candidate in logs. Record SHA-256 for every wheel, source SHA, CPU/emulation method, interpreter versions, wheel platform tag, libc floor, and audit output. The post-publication M004 tests are *not* part of this matrix. + +Stop rather than ship if ARMv7 wheel is not importable under matching emulation, contains incorrect hard-float ABI/GLIBC requirements, is falsely tagged, or the package's security policy breaks. Do not replace mandated ARMv7 with an unproven cross-compiled artifact. If unavoidable, register a corrective and obtain an explicit scoped decision before moving publication forward. Optional musllinux/i686 failures can be documented and deferred. + +## Closure evidence + +`plans/closure/python-pypi-distribution/002-status.md` records mandatory ARMv7 build/install/audit success and exact SHA/digest; each supplemental target as qualified or explicitly deferred; native vs emulator evidence; interpreter and libc floors; runtime fixture outcomes; source builds on SBC explicitly **not performed**; remaining hardware-model uncertainty classified *experimental*. Update roadmap and registry together when closing. diff --git a/plans/implementation/python-pypi-distribution/003-first-pypi-release-and-provenance.md b/plans/implementation/python-pypi-distribution/003-first-pypi-release-and-provenance.md new file mode 100644 index 0000000..4d287e8 --- /dev/null +++ b/plans/implementation/python-pypi-distribution/003-first-pypi-release-and-provenance.md @@ -0,0 +1,43 @@ +# M003 — First Controlled PyPI Publication and Installed-Wheel Provenance + +Status: active (workflow implemented; exact-SHA M001/M002 qualification passed; maintainer setup confirmed; awaiting default-branch workflow availability and publication) +Primary class: capability +Source: `plans/subsystems/python-pypi-distribution-roadmap.md` M003. +Qualified release candidate: `1135608af5d625cd72db6704adf3bcabd1a58ab1`, Eggsearch 0.4.4. Closure-only plan/registry updates followed at `65fba70705b2657bb40063283ccd51f28d2e993d`; they do not change the package source. Keep the candidate SHA explicit if merge strategy rewrites the commit. +Dependencies: interface — M001 `abi3-py311` / Python 3.15 metadata, M002 platform artifact contract; operational **publish blockers** — M001 and M002 closed on the exact candidate, clean audits, preflight owner/metadata validation, pre-created GitHub environment, and configured PyPI Trusted Publisher. Historical binding M008 is closed, but cannot qualify newer bytes. +Authorities: ADR-0005, `plans/003-planning-process.md`, `docs/release.md` (Rust/Eggpack release is separate), `plans/closure/python-native-library/008-status.md`. +User direction: the `eggsearch` PyPI name has been manually checked as available; proceed with first public distribution after the stated gates. No need to wait for post-publication SBC benchmarks. + +## Objective and current evidence + +Publish the first `eggsearch` Python wheel+sdist inventory to PyPI, allowing `pip`/`uv` consumers to install **prebuilt** artifacts for the stated supported platforms including experimental ARMv7 SBC. Protect immutability, exact-source provenance, architectural tag honesty, and dependency integrity. The user explicitly directs that a required reviewer not be configured; explicit workflow dispatch is the publication action. A published wheel and independently successful binary-only PyPI install, not a passing build, establish the user-visible capability. + +Current `bindings/python` is unpublished, `pyproject.toml` / Cargo manifests use version 0.4.4, and `.github/workflows/python-native-qualification.yml` is nonpublishing. Historical M008 candidate `067468a` qualified 0.4.3 with five wheels, 20 interpreter smokes, sdist rebuild and audit; new release bytes must be requalified. The root Rust publication sequence is controlled by Eggpack and remains unchanged. + +Execution added `.github/workflows/release-python-pypi.yml`. It accepts an exact candidate SHA, successful qualification run ID, and synchronized version; checks exact-SHA root CI and Python qualification; requires the pre-created `pypi` environment to exist; downloads only that run's artifacts; verifies the six-wheel-plus-sdist inventory and hashes; and publishes only through the `pypi` environment with OIDC permission scoped to the publish job. Post-publication binary-only consumer jobs cover five native platforms and emulated ARMv7. + +The selected candidate's M001/M002 acceptance is green: Python native qualification run `40729011504` passed all 34 jobs, root CI `40729011529` passed, egress qualification `40729042102` passed, and release drift run `40729011257` passed. The exact hosted artifact set was downloaded and accepted by `verify_release_artifacts.py`; hashes and the ARMv7 audit are recorded in `plans/closure/python-pypi-distribution/001-status.md` and `002-status.md`. No M003 release-workflow run or PyPI publication has been initiated. The maintainer confirmed that the PyPI Trusted Publisher and GitHub `pypi` environment are configured and directed that no required reviewer be used. The workflow verifies the environment exists before publication; manual workflow dispatch remains the explicit release action. + +## Non-goals and invariant boundaries + +No automatic publish on push or Rust tag, no rewrite of existing root releases or `release/eggpack/`, no Python wheel attachment to existing Eggpack 19-asset release contract, no bundled CLI/MCP subprocess, no unconditional source-build fallback on SBC, no physical-SBC benchmark gate, no PyPy/free-threaded claim, and no release based on a different SHA than the one whose artifacts were tested. + +Do not commit PyPI secrets or use long-lived API tokens when OIDC Trusted Publishing is available. Avoid mutable GitHub Action references; pin to approved 40-character commit SHAs and record upstream version. Keep permissions minimal and `id-token: write` **only in the manually dispatched publish workflow's publish job**. + +## Ordered work packages + +1. **Name, identity and version preflight.** Accept the user's manual check that the `eggsearch` project name is available; immediately before upload, independently verify current PyPI ownership/claim workflow, correct intended organization/maintainer and absence of collision or reserved-name policy rejection. Ensure first upload will claim the intended project, not publish to an unrelated distribution. Select the exact **unused** Python version from root/binding/pyproject synchronized versions; if 0.4.4 is still the version and free on PyPI, it is eligible; otherwise plan a coherent version bump, regenerate both locks and requalify. The user's instruction to publish now is authority to prepare a publishable release; any required repository or PyPI account setup/approval must remain explicit and auditable. +2. **Implement independent trusted-publishing job.** Add e.g. `.github/workflows/release-python-pypi.yml` as the sole Python writer; use explicit `workflow_dispatch` with exact 40-hex candidate SHA and chosen version, or an immutable approved release ref resolving to that SHA. A nonpublishing qualifier produces wheels/sdist for that SHA; publish job downloads only those exact artifacts from the associated run or reconstructs at that exact SHA **and requalifies rebuilt bytes**. Use a pre-created GitHub environment `pypi` and matching PyPI pending/normal trusted publisher configuration; fail closed if permissions unavailable, environment is missing, or source/ref mismatched. SHA-pin GitHub actions and PyPA publish action. Do not give OIDC to build jobs. +3. **Assemble and verify the artifact manifest.** Require exactly one sdist plus a documented wheel list: existing five mandatory families and qualified `manylinux_2_28_armv7l`, plus any M002-qualified optional musl/i686 families. Assert distribution name, exact version, metadata `Requires-Python >=3.11,<3.16`, `cp311-abi3`, wheel platform tags, unique tag/filename, license, README, `py.typed`, stub/module layout, source-lockfile completeness in the extracted sdist, absence of bundled Eggsearch executable/test transcript/credentials and no unexpected files. Record SHA-256, source SHA, build provenance, signed artifact attestations if available, and audit tool versions. Do not silently upload a subset when a mandatory wheel is missing. +4. **Qualify at the selected exact SHA.** Complete the M001 25 cell interpreter/platform matrix, M002 ARMv7 hosted QEMU installed-wheel smoke, optional musl smokes for included wheels, clean-venv wheel imports from outside the checkout with `--no-index --no-deps --only-binary=:all:`, offline sdist extraction/rebuild, Mode-Off/trust/cancellation and 10-tool checks, `cargo deny` on both lockfiles, Python build audit, `make check`, packaging guards and clean root `cargo publish --dry-run --locked`. Ensure qualifying workflow path/branches can be dispatched against the exact commit; record run IDs. Never use old 0.4.3 passing results as the release gate. +5. **Stage and publish.** Prefer TestPyPI/ephemeral package index trial if pending publisher and project namespaces allow without disrupting the imminent release; **TestPyPI is not a mandatory delay** if the entire verified wheelhouse was locally installed under matching platforms and PyPI OIDC credentials can be configured. The first real PyPI upload must be a deliberate `workflow_dispatch` using the exact verified inventory. A name or version conflict, partial upload or bad metadata must fail closed; never overwrite/delete immutable artifacts. Rebuild and requalify under a **new** version if a correction is required. +6. **Independent post-upload install proof — still M003.** In fresh hosted environments **without Cargo/Rust/CC**, install from the actual public PyPI index with a version pin and `--only-binary=:all:`; record `pip debug --verbose` (tags), `importlib.metadata` version, `eggsearch.tool_names()`, a bounded local/no-network fixture, and wheel URL/digest. Cover standard x86_64/aarch64 and macOS/Windows plus an ARMv7 QEMU userland, explicitly marking emulated evidence. Confirm PyPI page shows expected wheels/sdist and correct platform+Requires-Python metadata, and that `pip` does not fetch/build an sdist for tested SBC targets. +7. **Docs, governance, closure.** Update `docs/python-library.md`, `bindings/python/PYTHON_README.md`, `docs/release.md`, `README.md`, `architecture/python-library.md` and any Python-specific release checklist. Document PyPI install (`python -m pip install eggsearch` for normal hosts, `python -m pip install --only-binary=:all: eggsearch` for SBC), architecture/glibc floor, explicitly **experimental SBC compatibility**, unsupported targets, no on-device source builds, cancellation/Mode-Off/security semantics, SDK/MCP difference and provenance. Coordinate version descriptions so binary/Rust release and Python publication are separate. + +## Failure semantics and publication controls + +A successful wheel upload with missing mandatory target, a wheel whose platform tag does not match its linked binary, wrong source SHA, failed dependency audit, or lack of installed-consumer proof is a blocked release, not a conditional success. If PyPI name ownership or OIDC setup cannot be completed, leave release **blocked** and preserve build artifacts and exact evidence. PyPI versions are immutable: record partial uploads and choose a new release version for repairs, never attempt to replace bytes. If native ARMv7 hardware is untested but matching emulated installed-wheel acceptance passes, publication **may proceed** with explicit *experimental SBC* label. Do not expand that label to claim ARMv6/soft-float or unsupported glibc/musl environments. + +## Verification and closure requirements + +Run all M001/M002 checks at the chosen commit and complete an independently verified `pip install --only-binary=:all: eggsearch==` from **public PyPI**, with no compiler on consumers. Collect: exact source SHA/tag/ref; chosen version and name-ownership proof; final release inventory/hashes; M001/M002 qualification IDs and jobs; sdist extraction/audits; GitHub publish run and dispatch inputs, OIDC publisher/environment details without secrets; PyPI project/version URL; raw public wheel download/installed smoke evidence; wheel sizes; unsupported/experimental coverage statement; no SBC build or benchmark performed before release. Record in `plans/closure/python-pypi-distribution/003-status.md` with requirement-to-evidence matrix, classified residuals and recommendation. Update roadmap and registry in the same closure commit. **Only M003 publication closure** may declare Python package publicly available. diff --git a/plans/implementation/python-pypi-distribution/004-post-publication-sbc-benchmarking.md b/plans/implementation/python-pypi-distribution/004-post-publication-sbc-benchmarking.md new file mode 100644 index 0000000..272bc16 --- /dev/null +++ b/plans/implementation/python-pypi-distribution/004-post-publication-sbc-benchmarking.md @@ -0,0 +1,33 @@ +# M004 — Post-Publication SBC Runtime Benchmarking and Resource Policy + +Status: blocked (operational: M003 first PyPI publication must succeed) +Primary class: polish +Source: `plans/subsystems/python-pypi-distribution-roadmap.md` M004. +Planning baseline: `main@13fcac0b7ac0125d38e4e9775ebe1d62028fda72`. A standard-library benchmark harness is prepared at `bindings/python/benchmarks/sbc_runtime.py`; it verifies the installed native extension against the public PyPI pip-report and wheel digest, requires ten unique tool names, validates trusted local Mode-Off search results and malformed-input rejection, uses a loopback-only fetch fixture, and records repeated sequential asyncio event loops. Actual execution must pin the **published** `eggsearch` Python version and its installed wheel hashes. +Authority: ADR-0005, `architecture/python-library.md`, historical Python M002/M005 lifecycle guarantees and M003 release closure. +Primary rule: **publish first; measure on SBCs later; never compile Eggsearch or its wheel on the device.** + +## Objective and evidence context + +Characterize performance/footprint of the actual PyPI-installed native package on representative small ARM boards, validate SBC runtime behavior beyond CI QEMU smoke, then decide whether low-footprint policies should change. Physical-SBC benchmarking is explicitly **not a prerequisite** for M001–M003 or the first PyPI upload. The pre-release ARMv7 wheel remains correctly tagged and emulation-tested but publicly labeled *experimental* until hardware evidence exists. + +The current binding initializes one process-scoped four-worker Tokio runtime with up to eight blocking threads and caps process-wide in-flight operations at 32. These are not necessarily optimal for low-memory/low-core hardware; they are characterization hypotheses, not grounds to alter production code before gathering measurements. + +## Non-goals and invariants + +No device-local Rust build, `maturin develop`, Cargo source compilation, compiler installation or sdist fallback. No mandatory on-device wheel production or large benchmark dependency stack. No changing results/ranking, security policy, ten-tool public API, trust semantics, cancellation/close or default platform support to chase an incidental synthetic benchmark. No portable performance threshold invented without baseline data. Do not equate one Raspberry Pi or Le Potato result with every SBC. + +## Ordered work packages + +1. Select at least one native aarch64 Linux SBC and one ARMv7 32-bit Linux environment as hardware access allows (e.g., Pi 5-class aarch64, Le Potato/Raspberry Pi OS ARMv7 hard-float), recording `uname -m`, `/etc/os-release`, glibc or musl version, Python version, physical RAM, core count, storage, swap and thermal/power mode. Devices are consumer/benchmark hosts only. If ARMv7 hardware is temporarily unavailable, classify that evidence as **blocked post-release**, without retroactively blocking M003. +2. Install **only the exact published wheel**, `python -m pip install --only-binary=:all: eggsearch==`, confirm download URL/digest and `importlib.metadata`; instrument package import and each benchmark from a fresh process. Refuse a source distribution, ensure no Cargo/compiler invocation. Use a lightweight Python standard-library measurement harness (e.g., `time.perf_counter_ns`, `resource.getrusage`, `tracemalloc` for Python allocations, Linux `/proc/self/status`, `/proc//smaps_rollup` for native RSS/PSS); optional `psutil` is not required. +3. Measure cold import and first Tokio initialization separately, initialization with default vs explicit Mode-Off config, idle RSS/PSS, memory after sequential search/fetch fixtures, 1/2/4/8/16/32 concurrent bounded calls, cancellation/drain latency, per-client close, sequential `asyncio.run` loops, and heavy-output JSON conversion near defined budget caps. Prefer deterministic local/recorded fixtures; external live-provider performance is a separate optional observation with provider failure context. +4. Capture CPU time, wall-time distributions (median, p95 and spread across repeated runs), peak RSS/PSS, thread counts, process FD count, total memory headroom, disk wheel footprint and observable thermal throttling. Vary default process-concurrency realistically, keep tests short and bounded, and use the same scenario configuration when comparing ARMv7/aarch64. Report exact command and metadata, not just averaged headline numbers. +5. Analyze whether fixed four Tokio workers/eight blocking threads or 32-admission budget is unnecessarily expensive on 1–2-core/low-memory hardware. If evidence supports adaptation, propose the narrowest backwards-compatible config/runtime setting, explicit default behavior, bounded lower/upper values and tests proving continued operation on larger hosts. Treat tuning as a **separate corrective/optimization plan and patch release**, not an unreviewed change to the published baseline. +6. Run a basic physical-device installed-wheel compatibility smoke beyond the benchmark: ten tool names, representative local Mode-Off evidence, no remote calls, malformed input rejection, error taxonomy, cancellation, close and repeated loops. Collect any exceptions and SDK behavior divergence; release a correction only after exact new candidate qualification. Update SBC support notes based on verified board/OS/interpreter pairs, not on product-family inference. + +## Verification, stop conditions and closure + +Run benchmark script with published wheel outside checkout, with an enforced `--only-binary=:all:` install and compilation-absence guard; check source SHA/version and wheel digest against M003 closure. Fail the device benchmark, not the published release, on out-of-memory, architecture mismatch or missing native platform; record failures. Keep profiling fixtures network-free unless clearly marked diagnostic. Preserve test logs, versioned scenario JSON/CSV summaries, resource statistics and notebook-free reproducible scripts under the designated testing/benchmarks documentation rather than generated tracked debris. + +Closure `plans/closure/python-pypi-distribution/004-status.md` requires hardware/platform inventory, published wheel identity, commands, distributional data including RSS/PSS, first-import/first-operation latency and cancellation/thread metrics, correctness smoke, limitations, experimental-status recommendation and follow-up plans for any measured issues. Update roadmap and registry together. M004 may close without optimization if measurements show the current policy is adequate; evidence, not benchmark-driven refactoring, is the deliverable. diff --git a/plans/registry.md b/plans/registry.md index ae7da74..9bde7c5 100644 --- a/plans/registry.md +++ b/plans/registry.md @@ -43,11 +43,16 @@ Historical `phase-*.md` plans and pre-migration roadmaps are archived under | Dependency evidence hardening | closed | `plans/subsystems/dependency-evidence-hardening-roadmap.md` | M010 corrective closed | M001-M010 closed. Current control point `plans/closure/dependency-evidence-hardening/010-status.md`; implementation `23676cc`, CI `36253474005`. | | Repository security, supply-chain, and maintenance hardening | active | `plans/subsystems/repository-hardening-roadmap.md` | M001/M002/M007/M008/M009/M010 closed; M006 blocked | Scheduled dependency-security run `38290328560` succeeded; M010 records M001/M007 closure. M002 closed by M009; M006 remains externally blocked on chromiumoxide. | | Python native library for agents | closed (M001-M008 closed; unpublished by design) | `plans/subsystems/python-native-library-roadmap.md` | M008 closed at `067468a`, qualifying run `38641009246` (27/27 incl. audit) | ADR-0005 accepted. Runs `38522548258`, `38608502185`/`38611932990`/`38614592873`, `38637266178`, and `38638066765` are historical/deficiency/supporting evidence only. Publication separately authorized. | +| Python PyPI distribution and prebuilt SBC wheels | active (M001/M002 closed; M003 active; M004 blocked) | `plans/subsystems/python-pypi-distribution-roadmap.md` | M001/M002 closed at `1135608af5d625cd72db6704adf3bcabd1a58ab1`; qualification run `40729011504` and exact artifact verifier passed. M003 gated release workflow is implemented; M004 benchmark harness is prepared | Historical Python M001-M008 stay closed. Maintainer must configure the PyPI Trusted Publisher and GitHub `pypi` approval environment before M003 publication. Do not compile on SBCs; physical SBC benchmarking starts after publication. | ## Dependency-ready implementation plans | Subsystem | Milestone | Status | Implementation plan | Dependencies / handoff note | |---|---|---|---|---| +| Python PyPI distribution | M001 Python 3.15 / PyO3 ABI3 requalification | closed | `plans/implementation/python-pypi-distribution/001-python315-toolchain-and-abi3.md` | Closed at `1135608af5d625cd72db6704adf3bcabd1a58ab1`; closure `plans/closure/python-pypi-distribution/001-status.md`; exact-SHA run `40729011504` passed 25/25 smoke cells. No publication. | +| Python PyPI distribution | M002 hosted ARMv7 and Linux wheel expansion | closed | `plans/implementation/python-pypi-distribution/002-hosted-armv7-and-linux-wheel-expansion.md` | Closed at `1135608af5d625cd72db6704adf3bcabd1a58ab1`; closure `plans/closure/python-pypi-distribution/002-status.md`; hosted ELF/tag audit and CPython 3.11/3.15 QEMU installed-wheel tests passed. Optional musllinux deferred. **No SBC compilation**. | +| Python PyPI distribution | M003 controlled first PyPI release | **active** (publish gated) | `plans/implementation/python-pypi-distribution/003-first-pypi-release-and-provenance.md` | Candidate `1135608af5d625cd72db6704adf3bcabd1a58ab1` qualified (Python run `40729011504`, root CI `40729011529`, egress `40729042102`); exact six-wheel/sdist verifier passed. Requires PyPI Trusted Publisher and reviewed GitHub `pypi` environment setup before dispatch. | +| Python PyPI distribution | M004 physical SBC benchmarking after publication | **blocked** | `plans/implementation/python-pypi-distribution/004-post-publication-sbc-benchmarking.md` | Operational blocker: M003 public PyPI release. No pre-publication SBC build/benchmark. | | Eggpack release adoption | M001 seven-target Eggpack producer cutover | closed | `plans/implementation/eggpack-release-adoption/001-seven-target-eggpack-producer-cutover.md` | Paired producer plan `eggstack/eggpack: plans/implementation/ecosystem-adoption/003b-eggsearch-seven-target-eggpack-producer-cutover.md` (`325d44e`). Implementation `eabbf80`; run `37531104901`; draft `405157598`; provenance `37561960304`; published 2026-10-07. Eggpack production code was not edited | | CodeGG legacy search parity corrective | M001 keyless/source-specific provider parity | closed | `plans/implementation/codegg-legacy-search-parity/001-keyless-source-provider-parity.md` | Closed at `e9103b4`; closure `plans/closure/codegg-legacy-search-parity/001-status.md`. Added Wikipedia, arXiv, PubMed, HN Algolia, and GitHub repository discovery plus the additive `structured_api` provider kind; inventory 37 -> 42. | | CodeGG legacy search parity corrective | M002 credentialed provider parity | closed | `plans/implementation/codegg-legacy-search-parity/002-credentialed-provider-parity.md` | Closed at `5233315`; closure `plans/closure/codegg-legacy-search-parity/002-status.md`. Added opt-in `serpapi` and `kagi` (current v1, terms gate passed); inventory 42 -> 44. | @@ -104,7 +109,7 @@ Corrective closure sequence: `plans/implementation/dependency-evidence-hardening ## Current execution order and dependency gates -**Python native library gate:** ADR-0005 authorizes a narrow direct Rust SDK facade and Python PyO3 native package without broad Rust module stability. M001-M008 are closed at `067468a` with fully-green qualifying run `38641009246` (5/5 builds, 20/20 smokes, offline sdist rebuild, green audit, digests in `008-status.md`). Earlier runs are historical/deficiency/supporting evidence only. Publishing still needs separate authorization; Eggpack binary assets remain unchanged. +**Python native library gate:** ADR-0005 authorizes a narrow direct Rust SDK facade and Python PyO3 native package without broad Rust module stability. M001-M008 are closed at `067468a` with fully-green qualifying run `38641009246` (5/5 builds, 20/20 smokes, offline sdist rebuild, green audit, digests in `008-status.md`). Earlier runs are historical/deficiency/supporting evidence only. Initial Python qualification was nonpublishing; first PyPI publication is now separately planned under `python-pypi-distribution` M003 with explicit human release approval. Eggpack binary assets remain unchanged. **Tool-surface gate:** M001-M007 closed. Do not run evaluation against pre-consolidation bytes and present it as consolidation @@ -178,6 +183,7 @@ release that makes HTTP discovery / reqwest optional without regressing | CodeGG legacy search parity | closed; M001 through M005 closed | `plans/closure/codegg-legacy-search-parity/005-status.md`; prior control points `plans/closure/codegg-legacy-search-parity/004-status.md` and `003-status.md`; implementations `e9103b4` (M001), `5233315` (M002), `8e5ec75` (M003), `81a2e49` (M004 release-prep), `32769d6` (M005 fix-prep); historical provider-parity closure retained, new Mode-Off locality issue separately registered | | Mode-Off local repository dispatch corrective | closed; M001-M003 closed | M001 closure `001-status.md`; M003 hosted qualification `38476463235`; M002 immutable crate/tag/release `v0.4.3`, producer run `38480449328`, provenance run `38483437078`, and closure `002-status.md`. CodeGG paired requalification remains consumer-owned. | | Python native library | closed; M001-M008 closed | `plans/subsystems/python-native-library-roadmap.md`; closure records `001-status.md`–`008-status.md`; M008 closed at `067468a`, run `38641009246`; unpublished by design | +| Python PyPI distribution | active; M001/M002 closed, M003 active, M004 blocked | `plans/subsystems/python-pypi-distribution-roadmap.md`; M001/M002 closed at `1135608af5d625cd72db6704adf3bcabd1a58ab1` with exact-SHA hosted qualification; M003 publication waits on PyPI/GitHub setup; M004 benchmark harness is prepared and hardware evidence follows publication | ## Closure rule diff --git a/plans/subsystems/python-pypi-distribution-roadmap.md b/plans/subsystems/python-pypi-distribution-roadmap.md new file mode 100644 index 0000000..68ae52f --- /dev/null +++ b/plans/subsystems/python-pypi-distribution-roadmap.md @@ -0,0 +1,53 @@ +# Python PyPI Distribution and Prebuilt SBC Wheels Roadmap + +Status: active (M001/M002 closed; M003 active; M004 blocked on first PyPI publication and native-device access) +Primary outcome: publish the existing in-process `eggsearch` Python package promptly, with CPython 3.11–3.15 and broad precompiled platform wheels, explicitly experimental SBC support, then perform hardware benchmarking **after publication**. +Planning baseline: `main` `13fcac0b7ac0125d38e4e9775ebe1d62028fda72` (2026-10-28; root/binding/pyproject 0.4.4). Refresh the baseline at execution. +Authority: user confirmed the `eggsearch` PyPI project name is available by manual check and requested a release-first sequence without compiling on SBCs. +Owners: `bindings/python`, `.github/workflows/python-native-qualification.yml`, a separate Python publication workflow, Python documentation, and Python-only benchmarking. Not Eggpack. +References: `plans/000-long-term-specification.md` §§1, 6–9; `plans/001-terminology-and-domain-model.md`; `plans/002-long-term-roadmap.md`; `plans/003-planning-process.md`; accepted `plans/adrs/ADR-0005-python-native-library-boundary.md`; `architecture/python-library.md`; historical `plans/subsystems/python-native-library-roadmap.md` and `plans/closure/python-native-library/008-status.md`. + +## Boundary and decision + +The original Python native library M001–M008 is **closed**, not reopened. Its M008 historical qualifying evidence (`067468a`, 27/27 jobs on 0.4.3) does not qualify the current 0.4.4 or a future PyPI release candidate. This is a separate distribution/publication workstream. No Python subprocess/MCP-server fallback is introduced. The library continues to delegate the ten canonical operations through `src/sdk.rs` and the Rust MCP tool handlers. + +**Release first, hardware benchmarks later.** Build and publish cross-platform wheels on CI/hosted runners or cross-compilation containers. Never require a native SBC Rust toolchain, Cargo, C compiler, or wheel compilation for installation or pre-publication qualification. Where no native ARMv7 runner exists, use hosted QEMU/compatible emulation and explicitly mark the installed result as emulation-qualified. Mark ARM/SBC compatibility *experimental* until post-release native-device evidence is gathered. Do not describe unverified SBCs as tested. + +The initial PyPI upload is part of M003, not M004. M004 performance characterization, tuning and native hardware sampling **must not block first publication**. Only correctness, distribution integrity, security, and at-least-emulated target installation gates are release prerequisites. + +## Invariants and exclusions + +- Preserve the canonical ten-tool API, schema fingerprint, tool payloads, typed calls, trust markers, sanitization, SSRF/config/Mode-Off constraints, 32-call bounded admission semantics, Python cancellation/close guarantees, and import-side-effect-free behavior. +- Keep PyO3, `pyo3-async-runtimes`, and Maturin dependencies isolated to `bindings/python`; preserve root Rust 1.89 minimum and binary/CLI packaging, the Eggpack 19-asset contract, and `#![deny(unsafe_code)]` in the root. +- No false interpreter/platform tags, `--manylinux off` passing as publishable, ABI mismatches, undocumented glibc/musl requirements, or claims of PyPy/free-threaded Python/Windows ARM64/ARMv6 without qualification. +- Do not add a mandatory on-SBC build, native SBC benchmarking, or physical SBC hardware to the release gate. No Rust/cargo invokes on SBC during first-release preparation. Source distribution is an explicit developer/unsupported-target fallback, not the SBC installation method. +- PyPI name is user-verified as available, not a reason to delay writing plans; check ownership and exact version once immediately before first publication. Do not pre-announce PyPI availability until the public index serves the bytes. +- Do not create a second release artifact inventory inside Eggpack, automatically publish on root tags, or modify previously published immutable releases. +- Upstream provider services are not a dependency of deterministic CI smoke. + +## Compatibility and wheel policy + +Required first-release matrix: CPython 3.11, 3.12, 3.13, 3.14, 3.15 with `abi3-py311`, plus existing five native wheel families (Linux glibc x86_64/aarch64 at the demonstrated manylinux2014/2.17 floor, macOS x86_64/arm64, Windows x86_64) and **Linux ARMv7 hard-float** (`armv7-unknown-linux-gnueabihf` / PyPI `armv7l` tag). Execution targets `manylinux_2_28_armv7l` using the pinned cross image, then installs and tests in the official manylinux 2.31 ARMv7 image. Audit must confirm the actual tag and glibc floor; support is limited to ARMv7 hard-float and is not a generic all-Pi/all-ARM promise. Keep the ABI3 baseline; there is no need for per-CPython wheels for a given platform. + +Additional desirable hosted-built wheels: `musllinux_1_2` for Linux x86_64/aarch64, optionally ARMv7 and i686. M002 determines which can be qualified without destabilizing the mandatory first-release matrix. Do not silently advertise a target lacking a wheel. If any required target cannot be built and installed under controlled emulation/native CI, record a **release blocker**, not an unqualified substitute or a misleading tag; seek an explicit scoped decision rather than silently dropping ARMv7. + +The term *experimental SBC support* covers hardware-model/runtime/footprint uncertainty, not unvalidated ABI or broken wheel loading. CI must prove a correct, importable wheel under a matching target environment before it reaches PyPI. + +## Dependency graph, milestones, and exits + +| Milestone | Class | Status | Dependencies | Release-facing exit | +|---|---|---|---|---| +| M001 Python 3.15 and binding toolchain qualification | closed | hard: historical Python M008 closed | Closed at `1135608af5d625cd72db6704adf3bcabd1a58ab1`; exact-SHA qualification run `40729011504` passed all 25 CPython 3.11–3.15 cells across the five existing platform families. Closure: `plans/closure/python-pypi-distribution/001-status.md`. | +| M002 hosted ARMv7 and Linux wheel expansion | closed | interface: M001 keeps `abi3-py311` stable; operational integration before M003 | Closed at `1135608af5d625cd72db6704adf3bcabd1a58ab1`; hosted ARMv7 hard-float build, ELF audit and CPython 3.11/3.15 QEMU installs passed in run `40729011504`. No SBC compilation; optional musllinux remains deferred. Closure: `plans/closure/python-pypi-distribution/002-status.md`. | +| M003 controlled PyPI first publication | active (publication gated) | interface: established M001/M002 artifact contracts; operational: M001/M002 qualification and exact candidate | Trusted Publisher configured; exact-SHA wheel/sdist set passes audits and clean consumers; operator publishes `eggsearch` to PyPI and independently verifies public binary-only installs. Workflow and exact-SHA artifacts are ready; PyPI Trusted Publisher and GitHub `pypi` approval environment setup remain. | +| M004 post-publication SBC benchmarking and tuning decision | blocked | operational: successful M003 publication and native ARMv7/aarch64 device access | Native SBC installed-wheel benchmarks for memory, latency and concurrency; no source compilation; evidence-based recommendations/patch release only if needed. The standard-library harness is prepared. | + +M001 and M002 are closed against the exact release candidate; the M003 workflow implementation is ready. The **M003 publish action** is gated on maintainer configuration of the PyPI Trusted Publisher and the GitHub `pypi` approval environment. M004 benchmarking is deliberately scheduled **after** publication; a device not immediately available is not a retroactive blocker for M003. + +Implementation handoffs: `plans/implementation/python-pypi-distribution/001-python315-toolchain-and-abi3.md` through `004-post-publication-sbc-benchmarking.md`. Closure records go to `plans/closure/python-pypi-distribution/NNN-status.md`. Keep the registry aligned. + +## Global verification and risks + +Record exact full commit SHA, frozen version, wheel names/tags, hashes, host/emulator architecture, build/audit tools and CI run IDs. Run `make check`, `make packaging-check`, `cargo publish --dry-run --locked` when tree cleanliness permits, binding `pytest`/type checks, sdist extraction/rebuild, and actual installation from the *selected final* wheelhouse with `--only-binary=:all: --no-deps --no-index`. Runtime tests must cover all ten operations, configuration policy, no-network Mode-Off, lifecycle/cancellation and import without side effects. An experimental ARMv7 wheel must at least pass a release-installed smoke under emulation; full physical-SBC performance and long-duration soak are M004. + +Risks: cross-ABI interpreter discovery, glibc floor/correct `armv7l` tagging, host runner/emulator availability, PyO3 ABI compatibility with 3.15, and upstream audit findings. Any incompatibility must be visible as a blocked requirement with the exact failing evidence. Publication requires human-controlled approval (GitHub environment) and immutable PyPI version handling; a failed upload cannot be overwritten.