diff --git a/Cargo.lock b/Cargo.lock index f77e5544..9c30515c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1325,7 +1325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2011,7 +2011,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2953,14 +2953,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell", @@ -3414,7 +3414,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/architecture/hardening.md b/architecture/hardening.md index 6a938a6c..4c10845c 100644 --- a/architecture/hardening.md +++ b/architecture/hardening.md @@ -100,7 +100,7 @@ Field order is insignificant by design (one `write_str` per field under the enti ## Local filesystem containment -`validate_local_fetch_path()` (`src/core/local.rs`) rejects empty, absolute, traversal (`..`), hidden (unless `include_hidden`), skipped-directory (`target`, `node_modules`, `.git`), binary-extension, oversized, and non-file targets. `safe_open_relative()` (`src/meta/safe_open.rs`) then opens race-free: component-wise walking, NUL rejection, and on Linux descriptor-relative `openat2` with kernel-enforced `RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS` (plus `RESOLVE_NO_SYMLINKS` when `follow_symlinks = false`), falling back to `openat` with `O_NOFOLLOW` only where `openat2` is unavailable. Symlink escapes, intermediate symlinks, loops, root replacement, and concurrent modification during validation are all rejected or handled; reads after open enforce a hard byte cap. Never bypass these two functions with a direct filesystem read for operator-supplied paths. +`validate_local_fetch_path()` (`src/core/local.rs`) rejects empty, absolute, traversal (`..`), hidden (unless `include_hidden`), skipped-directory (`target`, `node_modules`, `.git`), binary-extension, oversized, and non-file targets. `safe_open_relative()` (`src/meta/safe_open.rs`) then walks component-wise with NUL rejection: on Linux descriptor-relative `openat2` with kernel-enforced `RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS` (plus `RESOLVE_NO_SYMLINKS` when `follow_symlinks = false`), falling back to `openat` with `O_NOFOLLOW` only where `openat2` is unavailable; on non-Unix every component is `symlink_metadata`-checked at walk time and links are rejected with `SymlinkDetected` when following is disabled. Symlink escapes, intermediate symlinks, loops, and root replacement are rejected fail-closed on every platform; the exact variant is platform-specific (Windows reports `SymlinkDetected` from the walk-time check, macOS reports `Io` where `O_NOFOLLOW` surfaces `ENOTDIR`, Linux rejects from the untouched `openat2`/`openat` path with a non-`SymlinkDetected` variant). The regression test pins fail-closed on unix and `SymlinkDetected` on Windows, never success. Reads after open enforce a hard byte cap. Only Linux (and the non-Linux Unix `O_NOFOLLOW` fallback) eliminates the validation-to-open race atomically: on Windows a check-to-open swap window remains, contained by the user-owned jail-root trust boundary and recorded in the owning closure record. Never bypass these two functions with a direct filesystem read for operator-supplied paths. `SafeOpenError` taxonomy (`Empty`, `AbsolutePath`, `PathTraversal`, `NullByte`, `NotFound`, `SymlinkDetected`, `NotAFile`, `FileTooLarge`, `FileContentLimitExceeded`, `SafeSymlinkFollowingUnsupported`, `RootOpenFailed`, `Io`) keeps every rejection machine-readable: callers map variants to retrieval attempts and absence kinds instead of string-matching messages. On non-Linux Unix and non-Unix platforms, `follow_symlinks = true` returns `SafeSymlinkFollowingUnsupported` rather than attempting containment without a race-safe primitive. diff --git a/architecture/local-workspace.md b/architecture/local-workspace.md index 0028aedd..afcd5168 100644 --- a/architecture/local-workspace.md +++ b/architecture/local-workspace.md @@ -144,15 +144,26 @@ directory file descriptor: - Non-Linux with `follow_symlinks=true`: `SafeSymlinkFollowingUnsupported` — there is no race-safe containment primitive, so the backend reports instead of approximating. +- Windows (`follow_symlinks=false`): every component is + `symlink_metadata`-checked at walk time and links are rejected with + `SymlinkDetected`; the root itself is followed, matching the unix root + policy. Pre-walk validation rejects empty, absolute, `..`-traversal, and null-byte components plus hidden components unless `include_hidden` is set. -The final descriptor is `fstat`-checked for regular file type and size -limits, eliminating TOCTOU races between validation and open. +On Linux the final descriptor is `fstat`-checked for regular file type +and size limits, eliminating TOCTOU races between validation and open; +on Windows the walk-time link checks leave a documented check-to-open +swap window (no atomic no-follow open in std), contained by the +user-owned jail-root trust boundary. Intermediate links are rejected +fail-closed everywhere; Windows reports `SymlinkDetected`, macOS +reports `Io` (`ENOTDIR` from `O_NOFOLLOW`), and Linux rejects from the +untouched `openat2`/`openat` path with a platform-specific variant. `SafeOpenError` covers `Empty`, `AbsolutePath`, `PathTraversal`, -`NullByte`, `NotFound`, `RootOpenFailed`, and -`SafeSymlinkFollowingUnsupported`; `safe_read_file()` layers bounded reads -on top. +`NullByte`, `NotFound`, `SymlinkDetected`, `NotAFile`, `FileTooLarge`, +`FileContentLimitExceeded`, `RootOpenFailed`, +`SafeSymlinkFollowingUnsupported`, and `Io`; `safe_read_file()` layers +bounded reads on top. --- diff --git a/architecture/python-library.md b/architecture/python-library.md index adef198e..2bcf7125 100644 --- a/architecture/python-library.md +++ b/architecture/python-library.md @@ -9,8 +9,15 @@ Current handoff: M001's shared Rust facade and ten-operation parity are closed at `plans/closure/python-native-library/001-status.md`; M002's native asyncio bridge is closed at `plans/closure/python-native-library/002-status.md`; M003's Python API, types, examples, and semantic parity work is closed at -`plans/closure/python-native-library/003-status.md`; M004 distribution -qualification is active. +`plans/closure/python-native-library/003-status.md`; M005 in-flight +cancellation and default-config corrective is closed at +`plans/closure/python-native-library/005-status.md`; M004 distribution +qualification, M006 exact-SHA closure, and the M007 Windows safe-read +corrective are closed together at `plans/closure/python-native-library/004-status.md` +(candidate `faa1da2`, qualifying run `38617598636`, unpublished by design), +superseded as the qualification basis by M008 advisory remediation and +symlink containment at `plans/closure/python-native-library/008-status.md` +(candidate `067468a`, fully-green run `38641009246`). ## Rust operation facade @@ -40,6 +47,17 @@ in-flight calls at 32, and bounds JSON conversion to 1 MiB, 64 levels, and waits for their Rust futures to drop; Python task cancellation drops its corresponding future. The runtime intentionally remains process-scoped. +`AsyncEggsearch()` preserves no-arg `AppConfig::default()` semantics. +`AsyncEggsearch.from_default_config()` and `default_config_path()` use +Rust-owned `eggsearch::core::config::default_config_path()` plus +`AppConfig::load`, preserving explicit-path behavior and CLI missing-file +fallback without reimplementing platform selection in Python. Cancellation +after a started barrier is proven by a `src/sdk.rs` counted blocking engine +plus a debug-only bridge gate (`_test_blocking_execute`, `_test_gate_state`, +`_test_gate_reset`, `_test_gate_release`, `_test_active_count`) that shares the +production `ActiveCall`, 32-permit, and close-token path; the gate is compiled +out with `debug_assertions` off and never ships in release wheels. + Milestone status and exact-candidate evidence live in `plans/registry.md` and `plans/closure/python-native-library/`. A compiled extension alone is not evidence of a supported Python consumer. Wheel support diff --git a/architecture/testing.md b/architecture/testing.md index 4e4a7b6c..3531ab27 100644 --- a/architecture/testing.md +++ b/architecture/testing.md @@ -24,7 +24,7 @@ Per-suite counts live in `docs/test-inventory.md`; suite placement rules live in | `tests/mcp_http.rs` | Loopback Streamable HTTP lifecycle and transport hardening | | `packaging/test-install.sh`, `packaging/test-install.ps1` | Installer behavior (not `tests/`) | | `src/**/mod.rs` unit tests | Private-function coverage at the bottom of the source file | -| `bindings/python/tests/test_native.py` | PyO3 asyncio consumer behavior, typed method delegation, bounded input conversion, cancellation and close lifecycle | +| `bindings/python/tests/test_native.py` | PyO3 asyncio consumer behavior, typed method delegation, bounded input conversion, started-barrier cancellation with Rust drop/permit recovery, concurrent close and 32-call saturation recovery, conversion-failure leak checks, sequential loop recovery, and default-config selection | --- diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 073cb670..5c053a0d 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -1837,9 +1837,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell", diff --git a/bindings/python/PYTHON_README.md b/bindings/python/PYTHON_README.md index 20ad7671..e0e14218 100644 --- a/bindings/python/PYTHON_README.md +++ b/bindings/python/PYTHON_README.md @@ -31,5 +31,12 @@ async def main() -> None: asyncio.run(main()) ``` +`AsyncEggsearch.from_default_config()` loads the standard installed +`eggsearch/config.toml` via Rust-owned path resolution, matching CLI fallback +semantics. Bare `AsyncEggsearch()` keeps `AppConfig` defaults and never reads +that path implicitly. Cancelling a pending operation drops its Rust future and +releases the shared 32-call permit; `aclose()` drains active work and reports +`client_closed` for close-triggered calls. + See [`docs/python-library.md`](../../docs/python-library.md) for configuration, types, trust handling, and the framework-neutral search/fetch/evidence example. diff --git a/bindings/python/python/eggsearch/__init__.py b/bindings/python/python/eggsearch/__init__.py index 56f807c9..06cae0f6 100644 --- a/bindings/python/python/eggsearch/__init__.py +++ b/bindings/python/python/eggsearch/__init__.py @@ -38,6 +38,16 @@ def __init__(self, config_path: str | Path | None = None) -> None: None if config_path is None else str(config_path) ) + @classmethod + def from_default_config(cls) -> AsyncEggsearch: + instance = cls.__new__(cls) + instance._native = _NativeAsyncEggsearch.from_default_config() + return instance + + @staticmethod + def default_config_path() -> str: + return str(_NativeAsyncEggsearch.default_config_path()) + async def __aenter__(self) -> AsyncEggsearch: return self diff --git a/bindings/python/python/eggsearch/_native.pyi b/bindings/python/python/eggsearch/_native.pyi index 1bc8e892..c7e24a5f 100644 --- a/bindings/python/python/eggsearch/_native.pyi +++ b/bindings/python/python/eggsearch/_native.pyi @@ -10,6 +10,10 @@ class EggsearchError(Exception): class AsyncEggsearch: def __init__(self, config_path: str | None = ...) -> None: ... + @staticmethod + def default_config_path() -> str: ... + @staticmethod + def from_default_config() -> AsyncEggsearch: ... def execute(self, tool: str, arguments: dict[str, Any]) -> Any: ... def aclose(self) -> Any: ... def tool_contracts(self) -> Any: ... diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 40824fa1..20d39ee5 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -1,5 +1,7 @@ use std::collections::HashSet; use std::path::PathBuf; +#[cfg(debug_assertions)] +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, LazyLock, Mutex, OnceLock}; use eggsearch::core::config::AppConfig; @@ -23,6 +25,40 @@ create_exception!(_native, EggsearchError, pyo3::exceptions::PyException); static RUNTIME_INIT: OnceLock<()> = OnceLock::new(); static IN_FLIGHT: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(MAX_IN_FLIGHT))); +#[cfg(debug_assertions)] +static TEST_GATE: LazyLock = LazyLock::new(TestGate::new); +#[cfg(debug_assertions)] +struct TestGate { + entered: AtomicUsize, + dropped: AtomicUsize, + release_count: AtomicUsize, + release: Notify, +} +#[cfg(debug_assertions)] +impl TestGate { + fn new() -> Self { + Self { + entered: AtomicUsize::new(0), + dropped: AtomicUsize::new(0), + release_count: AtomicUsize::new(0), + release: Notify::new(), + } + } + fn reset(&self) { + self.entered.store(0, Ordering::SeqCst); + self.dropped.store(0, Ordering::SeqCst); + self.release_count.fetch_add(1, Ordering::SeqCst); + self.release.notify_waiters(); + } +} +#[cfg(debug_assertions)] +struct GateDropGuard; +#[cfg(debug_assertions)] +impl Drop for GateDropGuard { + fn drop(&mut self) { + TEST_GATE.dropped.fetch_add(1, Ordering::SeqCst); + } +} fn initialize_runtime() { RUNTIME_INIT.get_or_init(|| { @@ -121,27 +157,50 @@ struct PyAsyncEggsearch { state: Arc, } +fn build_state(config: AppConfig) -> PyResult> { + let client = EggsearchClient::new(config).map_err(|error| { + PyValueError::new_err(format!("invalid Eggsearch configuration: {error}")) + })?; + Ok(Arc::new(ClientState { + client, + lifecycle: Mutex::new(Lifecycle::default()), + cancelled: CancellationToken::new(), + drained: Notify::new(), + })) +} + +fn load_config_from_path(path: &PathBuf) -> PyResult { + AppConfig::load(path) + .map_err(|error| PyValueError::new_err(format!("invalid Eggsearch configuration: {error}"))) +} + #[pymethods] impl PyAsyncEggsearch { #[new] #[pyo3(signature = (config_path=None))] fn new(config_path: Option) -> PyResult { let config = match config_path { - Some(path) => AppConfig::load(&path).map_err(|error| { - PyValueError::new_err(format!("invalid Eggsearch configuration: {error}")) - })?, + Some(path) => load_config_from_path(&path)?, None => AppConfig::default(), }; - let client = EggsearchClient::new(config).map_err(|error| { - PyValueError::new_err(format!("invalid Eggsearch configuration: {error}")) - })?; Ok(Self { - state: Arc::new(ClientState { - client, - lifecycle: Mutex::new(Lifecycle::default()), - cancelled: CancellationToken::new(), - drained: Notify::new(), - }), + state: build_state(config)?, + }) + } + + #[staticmethod] + fn default_config_path() -> String { + eggsearch::core::config::default_config_path() + .to_string_lossy() + .into_owned() + } + + #[staticmethod] + fn from_default_config() -> PyResult { + let path = eggsearch::core::config::default_config_path(); + let config = load_config_from_path(&path)?; + Ok(Self { + state: build_state(config)?, }) } @@ -214,6 +273,102 @@ impl PyAsyncEggsearch { let contracts = self.state.client.tool_contracts(); json_to_python(py, &contracts).map(Bound::unbind) } + + #[cfg(debug_assertions)] + #[staticmethod] + fn _test_gate_reset() { + TEST_GATE.reset(); + } + + #[cfg(debug_assertions)] + #[staticmethod] + fn _test_gate_release() { + TEST_GATE.release_count.fetch_add(1, Ordering::SeqCst); + TEST_GATE.release.notify_waiters(); + } + + #[cfg(debug_assertions)] + #[staticmethod] + fn _test_gate_state(py: Python<'_>) -> PyResult> { + let snapshot = serde_json::json!({ + "entered": TEST_GATE.entered.load(Ordering::SeqCst), + "dropped": TEST_GATE.dropped.load(Ordering::SeqCst), + "release_count": TEST_GATE.release_count.load(Ordering::SeqCst), + "permits_available": IN_FLIGHT.available_permits(), + }); + json_to_python(py, &snapshot).map(Bound::unbind) + } + + #[cfg(debug_assertions)] + fn _test_active_count(&self) -> usize { + self.state + .lifecycle + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .active + } + + #[cfg(debug_assertions)] + fn _test_blocking_execute<'py>(&self, py: Python<'py>) -> PyResult> { + initialize_runtime(); + let active_call = self + .state + .register() + .map_err(|error| sdk_error(py, error))?; + let permit = IN_FLIGHT.clone().try_acquire_owned().map_err(|_| { + sdk_error( + py, + SdkError { + code: "budget_invalid".into(), + message: "native execution concurrency limit reached".into(), + repair: None, + data: None, + }, + ) + })?; + let state = self.state.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let _active_call = active_call; + let _permit: OwnedSemaphorePermit = permit; + let _gate_guard = GateDropGuard; + TEST_GATE.entered.fetch_add(1, Ordering::SeqCst); + let start = TEST_GATE.release_count.load(Ordering::SeqCst); + let result = std::panic::AssertUnwindSafe(async { + loop { + tokio::select! { + _ = state.cancelled.cancelled() => { + return Err(SdkError { + code: "client_closed".into(), + message: "Eggsearch client closed while the operation was running" + .into(), + repair: None, + data: None, + }); + } + _ = TEST_GATE.release.notified() => { + if TEST_GATE.release_count.load(Ordering::SeqCst) != start { + return Ok(serde_json::json!({"gate": "released"})); + } + } + } + } + }) + .catch_unwind() + .await + .unwrap_or_else(|_| { + Err(SdkError { + code: "internal".into(), + message: "internal native operation failure".into(), + repair: None, + data: None, + }) + }); + Python::attach(|py| match result { + Ok(value) => json_to_python(py, &value).map(Bound::unbind), + Err(error) => Err(sdk_error(py, error)), + }) + }) + } } #[pyfunction] diff --git a/bindings/python/tests/test_native.py b/bindings/python/tests/test_native.py index 1006b291..ae1806f0 100644 --- a/bindings/python/tests/test_native.py +++ b/bindings/python/tests/test_native.py @@ -1,5 +1,7 @@ import asyncio +import json import math +import os import pytest @@ -7,6 +9,58 @@ from eggsearch import types as eggsearch_types +def _require_gate(): + native = AsyncEggsearch()._native + if not hasattr(native, "_test_blocking_execute"): + pytest.skip("native test gate is debug-only and absent in release builds") + + +async def _wait_gate(native_cls, want_entered, timeout=5.0): + async def _poll(): + while True: + state = native_cls._test_gate_state() + if state["entered"] >= want_entered: + return state + await asyncio.sleep(0.005) + + return await asyncio.wait_for(_poll(), timeout=timeout) + + +async def _wait_gate_dropped(native_cls, want_dropped, timeout=5.0): + async def _poll(): + while True: + state = native_cls._test_gate_state() + if state["dropped"] >= want_dropped: + return state + await asyncio.sleep(0.005) + + return await asyncio.wait_for(_poll(), timeout=timeout) + + +def _platform_env(monkeypatch, base): + monkeypatch.setenv("XDG_CONFIG_HOME", str(base)) + monkeypatch.setenv("HOME", str(base)) + monkeypatch.setenv("APPDATA", str(base)) + monkeypatch.setenv("USERPROFILE", str(base)) + + +def _write_config(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _mojeek_enabled(status): + providers = status.get("providers", status) + if isinstance(providers, dict): + providers = providers.get("providers", []) + for entry in providers if isinstance(providers, list) else []: + if not isinstance(entry, dict): + continue + if entry.get("id") == "mojeek" or entry.get("provider_id") == "mojeek": + return bool(entry.get("enabled")) + return None + + EXPECTED_TOOLS = { "web_search", "web_fetch", @@ -292,3 +346,610 @@ async def run(): assert bundle["sources"][0].get("trust") == "local_trusted" asyncio.run(run()) + + +def test_native_gate_cancel_after_started_barrier_drops_rust_future(config_path): + _require_gate() + + async def run(): + native_cls = type(AsyncEggsearch(str(config_path))._native) + native_cls._test_gate_reset() + client = AsyncEggsearch(str(config_path)) + native = client._native + assert native._test_active_count() == 0 + pending = native._test_blocking_execute() + try: + state = await _wait_gate(native_cls, 1) + assert state["entered"] == 1 + assert state["dropped"] == 0 + assert native._test_active_count() == 1 + assert state["permits_available"] == 31 + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(pending, timeout=5.0) + state = await _wait_gate_dropped(native_cls, 1) + assert state["entered"] == 1 + assert state["dropped"] == 1 + async def _wait_idle(): + while True: + if native._test_active_count() == 0: + current = native_cls._test_gate_state() + if current["permits_available"] == 32: + return current + await asyncio.sleep(0.005) + + state = await asyncio.wait_for(_wait_idle(), timeout=5.0) + assert native._test_active_count() == 0 + assert state["permits_available"] == 32 + status = await asyncio.wait_for( + client.execute("provider_status", {}), timeout=5.0 + ) + assert isinstance(status, dict) + finally: + native_cls._test_gate_release() + try: + await asyncio.wait_for(pending, timeout=2.0) + except (asyncio.CancelledError, Exception): + pass + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(run()) + + +def test_native_gate_close_cancels_inflight_with_client_closed(config_path): + _require_gate() + + async def run(): + probe = AsyncEggsearch(str(config_path)) + native_cls = type(probe._native) + await probe.aclose() + native_cls._test_gate_reset() + client = AsyncEggsearch(str(config_path)) + native = client._native + first_task = asyncio.ensure_future(native._test_blocking_execute()) + second_task = asyncio.ensure_future(native._test_blocking_execute()) + try: + await _wait_gate(native_cls, 2) + assert native._test_active_count() == 2 + await asyncio.wait_for(client.aclose(), timeout=5.0) + for task in (first_task, second_task): + with pytest.raises(EggsearchError) as closed: + await asyncio.wait_for(task, timeout=5.0) + assert closed.value.code == "client_closed" + state = await _wait_gate_dropped(native_cls, 2) + assert state["entered"] == 2 + assert state["dropped"] == 2 + assert native._test_active_count() == 0 + assert state["permits_available"] == 32 + await asyncio.wait_for(client.aclose(), timeout=5.0) + with pytest.raises(EggsearchError) as rejected: + await client.provider_status() + assert rejected.value.code == "client_closed" + fresh = AsyncEggsearch(str(config_path)) + try: + status = await asyncio.wait_for( + fresh.execute("provider_status", {}), timeout=5.0 + ) + assert isinstance(status, dict) + finally: + await asyncio.wait_for(fresh.aclose(), timeout=5.0) + finally: + native_cls._test_gate_release() + for task in (first_task, second_task): + if not task.done(): + task.cancel() + try: + await asyncio.wait_for(task, timeout=2.0) + except (asyncio.CancelledError, Exception): + pass + try: + await asyncio.wait_for(client.aclose(), timeout=5.0) + except Exception: + pass + + asyncio.run(run()) + + +def test_native_gate_concurrency_cap_saturates_and_recovers(config_path): + _require_gate() + + async def run(): + probe = AsyncEggsearch(str(config_path)) + native_cls = type(probe._native) + await probe.aclose() + native_cls._test_gate_reset() + client = AsyncEggsearch(str(config_path)) + other = AsyncEggsearch(str(config_path)) + tasks = [] + try: + for _ in range(32): + tasks.append(asyncio.ensure_future(client._native._test_blocking_execute())) + await _wait_gate(native_cls, 32) + state = native_cls._test_gate_state() + assert state["entered"] == 32 + assert state["permits_available"] == 0 + with pytest.raises(EggsearchError) as saturated: + await client._native._test_blocking_execute() + assert saturated.value.code == "budget_invalid" + with pytest.raises(EggsearchError) as other_saturated: + await other.execute("provider_status", {}) + assert other_saturated.value.code == "budget_invalid" + native_cls._test_gate_release() + results = await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), timeout=10.0 + ) + assert len(results) == 32 + assert all(isinstance(value, dict) for value in results) + state = await _wait_gate_dropped(native_cls, 32) + assert state["dropped"] == 32 + async def _wait_permits(): + while True: + current = native_cls._test_gate_state() + if current["permits_available"] == 32: + return current + await asyncio.sleep(0.005) + + state = await asyncio.wait_for(_wait_permits(), timeout=5.0) + assert state["permits_available"] == 32 + assert client._native._test_active_count() == 0 + status = await asyncio.wait_for( + client.execute("provider_status", {}), timeout=5.0 + ) + assert isinstance(status, dict) + other_status = await asyncio.wait_for( + other.execute("provider_status", {}), timeout=5.0 + ) + assert isinstance(other_status, dict) + finally: + native_cls._test_gate_release() + if tasks: + try: + await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), timeout=5.0 + ) + except Exception: + pass + await asyncio.wait_for(client.aclose(), timeout=5.0) + await asyncio.wait_for(other.aclose(), timeout=5.0) + + asyncio.run(run()) + + +def test_native_conversion_failures_do_not_leak_permits(config_path): + _require_gate() + + async def run(): + probe = AsyncEggsearch(str(config_path)) + native_cls = type(probe._native) + await probe.aclose() + native_cls._test_gate_reset() + client = AsyncEggsearch(str(config_path)) + try: + assert client._native._test_active_count() == 0 + assert native_cls._test_gate_state()["permits_available"] == 32 + with pytest.raises((TypeError, ValueError)): + client._native.execute("web_search", []) + assert client._native._test_active_count() == 0 + assert native_cls._test_gate_state()["permits_available"] == 32 + value = [] + value.append(value) + with pytest.raises(ValueError, match="cyclic"): + client._native.execute("web_search", {"value": value}) + assert client._native._test_active_count() == 0 + assert native_cls._test_gate_state()["permits_available"] == 32 + status = await asyncio.wait_for( + client.execute("provider_status", {}), timeout=5.0 + ) + assert isinstance(status, dict) + finally: + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(run()) + + +def test_native_runtime_recovers_on_next_loop_after_cancel_close(config_path): + _require_gate() + + async def first(): + native_cls = type(AsyncEggsearch(str(config_path))._native) + native_cls._test_gate_reset() + client = AsyncEggsearch(str(config_path)) + pending = client._native._test_blocking_execute() + try: + await _wait_gate(native_cls, 1) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(pending, timeout=5.0) + finally: + native_cls._test_gate_release() + try: + await asyncio.wait_for(pending, timeout=2.0) + except (asyncio.CancelledError, Exception): + pass + await asyncio.wait_for(client.aclose(), timeout=5.0) + + async def second(): + client = AsyncEggsearch(str(config_path)) + try: + status = await asyncio.wait_for( + client.execute("provider_status", {}), timeout=5.0 + ) + assert isinstance(status, dict) + finally: + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(first()) + asyncio.run(second()) + + +def test_no_arg_defaults_ignore_platform_config(tmp_path, monkeypatch): + isolated = tmp_path / "isolated-home" + isolated.mkdir() + _platform_env(monkeypatch, isolated) + monkeypatch.chdir(tmp_path) + resolved = AsyncEggsearch.default_config_path() + assert resolved + assert "eggsearch" in resolved + assert resolved.endswith("config.toml") or resolved.endswith("eggsearch.toml") + _write_config( + __import__("pathlib").Path(resolved), + "[search]\n" + "mode = 'off'\n" + "default_max_results = 10\n" + "max_results_cap = 50\n" + "max_query_chars = 512\n" + "timeout_ms = 8000\n" + "default_providers = ['duckduckgo']\n" + "sanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\nmojeek = true\n\n" + "[fetch]\n" + "enabled = false\n" + "timeout_ms = 8000\n" + "max_bytes = 2000000\n" + "max_chars_default = 12000\n" + "max_chars_cap = 50000\n" + "redirect_limit = 5\n" + "allow_private_network = false\n" + "allow_localhost = false\n" + "sanitize_output = true\n\n" + "[local]\nenabled = false\nroots = []\n", + ) + + async def run(): + client = AsyncEggsearch() + try: + status = await asyncio.wait_for( + client.provider_status(probe=False), timeout=5.0 + ) + assert _mojeek_enabled(status) is False + finally: + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(run()) + + +def test_from_default_config_loads_platform_config(tmp_path, monkeypatch): + workspace = tmp_path / "platform-workspace" + workspace.mkdir() + (workspace / "README.md").write_text( + "Distinctive platform-config corrective fixture for native Python.", + encoding="utf-8", + ) + isolated = tmp_path / "isolated-home" + isolated.mkdir() + _platform_env(monkeypatch, isolated) + monkeypatch.chdir(tmp_path) + resolved = AsyncEggsearch.default_config_path() + _write_config( + __import__("pathlib").Path(resolved), + "[search]\n" + "mode = 'off'\n" + "default_max_results = 10\n" + "max_results_cap = 50\n" + "max_query_chars = 512\n" + "timeout_ms = 8000\n" + "default_providers = ['duckduckgo']\n" + "sanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\nmojeek = true\n\n" + "[fetch]\n" + "enabled = false\n" + "timeout_ms = 8000\n" + "max_bytes = 2000000\n" + "max_chars_default = 12000\n" + "max_chars_cap = 50000\n" + "redirect_limit = 5\n" + "allow_private_network = false\n" + "allow_localhost = false\n" + "sanitize_output = true\n\n" + f"[local]\nenabled = true\nroots = [\"{workspace.as_posix()}\"]\n", + ) + + async def run(): + client = AsyncEggsearch.from_default_config() + try: + status = await asyncio.wait_for( + client.provider_status(probe=False), timeout=5.0 + ) + assert _mojeek_enabled(status) is True + result = await asyncio.wait_for( + client.repo_search(query="corrective", include_local=True), + timeout=10.0, + ) + cards = [ + card + for group in result.get("groups", []) + for card in group.get("results", []) + ] + assert cards + assert any(card.get("trust") == "local_trusted" for card in cards) + finally: + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(run()) + + +def test_explicit_path_overrides_default_config(tmp_path, monkeypatch): + default_workspace = tmp_path / "default-workspace" + default_workspace.mkdir() + (default_workspace / "README.md").write_text( + "Default-path workspace marker alpha.", + encoding="utf-8", + ) + explicit_workspace = tmp_path / "explicit-workspace" + explicit_workspace.mkdir() + (explicit_workspace / "README.md").write_text( + "Explicit-path workspace marker beta.", + encoding="utf-8", + ) + isolated = tmp_path / "isolated-home" + isolated.mkdir() + _platform_env(monkeypatch, isolated) + monkeypatch.chdir(tmp_path) + resolved = AsyncEggsearch.default_config_path() + _write_config( + __import__("pathlib").Path(resolved), + "[search]\n" + "mode = 'off'\n" + "default_max_results = 10\n" + "max_results_cap = 50\n" + "max_query_chars = 512\n" + "timeout_ms = 8000\n" + "default_providers = ['duckduckgo']\n" + "sanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\nmojeek = true\n\n" + "[fetch]\n" + "enabled = false\n" + "timeout_ms = 8000\n" + "max_bytes = 2000000\n" + "max_chars_default = 12000\n" + "max_chars_cap = 50000\n" + "redirect_limit = 5\n" + "allow_private_network = false\n" + "allow_localhost = false\n" + "sanitize_output = true\n\n" + f"[local]\nenabled = true\nroots = [\"{default_workspace.as_posix()}\"]\n", + ) + explicit_path = tmp_path / "explicit.toml" + explicit_path.write_text( + "[search]\n" + "mode = 'off'\n" + "default_max_results = 10\n" + "max_results_cap = 50\n" + "max_query_chars = 512\n" + "timeout_ms = 8000\n" + "default_providers = ['duckduckgo']\n" + "sanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\nmojeek = false\n\n" + "[fetch]\n" + "enabled = false\n" + "timeout_ms = 8000\n" + "max_bytes = 2000000\n" + "max_chars_default = 12000\n" + "max_chars_cap = 50000\n" + "redirect_limit = 5\n" + "allow_private_network = false\n" + "allow_localhost = false\n" + "sanitize_output = true\n\n" + f"[local]\nenabled = true\nroots = [\"{explicit_workspace.as_posix()}\"]\n", + encoding="utf-8", + ) + + async def run(): + defaulted = AsyncEggsearch.from_default_config() + explicit = AsyncEggsearch(str(explicit_path)) + try: + default_status = await asyncio.wait_for( + defaulted.provider_status(probe=False), timeout=5.0 + ) + explicit_status = await asyncio.wait_for( + explicit.provider_status(probe=False), timeout=5.0 + ) + assert _mojeek_enabled(default_status) is True + assert _mojeek_enabled(explicit_status) is False + default_result = await asyncio.wait_for( + defaulted.repo_search(query="alpha", include_local=True), + timeout=10.0, + ) + explicit_result = await asyncio.wait_for( + explicit.repo_search(query="beta", include_local=True), + timeout=10.0, + ) + assert any( + "alpha" in (card.get("snippet") or "") + for group in default_result.get("groups", []) + for card in group.get("results", []) + ) + assert any( + "beta" in (card.get("snippet") or "") + for group in explicit_result.get("groups", []) + for card in group.get("results", []) + ) + finally: + await asyncio.wait_for(defaulted.aclose(), timeout=5.0) + await asyncio.wait_for(explicit.aclose(), timeout=5.0) + + asyncio.run(run()) + + +@pytest.mark.skipif(os.name != "nt", reason="Windows non-Unix safe-open path only") +def test_windows_symlinked_local_file_content_not_surfaced(tmp_path, monkeypatch): + workspace = tmp_path / "symlink-workspace" + workspace.mkdir() + (workspace / "README.md").write_text( + "Real workspace marker gamma.", + encoding="utf-8", + ) + outside = tmp_path / "outside.md" + outside.write_text( + "Outside symlink target marker zeta.", + encoding="utf-8", + ) + try: + os.symlink(outside, workspace / "link.md") + except OSError: + pytest.skip("symlink creation privilege unavailable") + isolated = tmp_path / "symlink-home" + isolated.mkdir() + _platform_env(monkeypatch, isolated) + monkeypatch.chdir(tmp_path) + config = tmp_path / "symlink.toml" + config.write_text( + "[search]\n" + "mode = 'off'\n" + "default_max_results = 10\n" + "max_results_cap = 50\n" + "max_query_chars = 512\n" + "timeout_ms = 8000\n" + "default_providers = ['duckduckgo']\n" + "sanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\nmojeek = true\n\n" + "[fetch]\n" + "enabled = false\n" + "timeout_ms = 8000\n" + "max_bytes = 2000000\n" + "max_chars_default = 12000\n" + "max_chars_cap = 50000\n" + "redirect_limit = 5\n" + "allow_private_network = false\n" + "allow_localhost = false\n" + "sanitize_output = true\n\n" + f"[local]\nenabled = true\nroots = [\"{workspace.as_posix()}\"]\n", + encoding="utf-8", + ) + + async def run(): + client = AsyncEggsearch(str(config)) + try: + gamma_result = await asyncio.wait_for( + client.repo_search(query="gamma", include_local=True), + timeout=10.0, + ) + zeta_result = await asyncio.wait_for( + client.repo_search(query="zeta", include_local=True), + timeout=10.0, + ) + snippets = [ + card.get("snippet") or "" + for result in (gamma_result, zeta_result) + for group in result.get("groups", []) + for card in group.get("results", []) + ] + assert any("gamma" in snippet for snippet in snippets) + assert all("zeta" not in snippet for snippet in snippets) + finally: + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(run()) + + +def test_from_default_config_missing_file_falls_back_to_defaults( + tmp_path, monkeypatch +): + isolated = tmp_path / "empty-home" + isolated.mkdir() + _platform_env(monkeypatch, isolated) + monkeypatch.chdir(tmp_path) + + async def run(): + client = AsyncEggsearch.from_default_config() + try: + status = await asyncio.wait_for( + client.provider_status(probe=False), timeout=5.0 + ) + assert isinstance(status, dict) + assert _mojeek_enabled(status) is False + finally: + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(run()) + + +def test_from_default_config_invalid_toml_reports_deterministic_error( + tmp_path, monkeypatch +): + isolated = tmp_path / "bad-home" + isolated.mkdir() + _platform_env(monkeypatch, isolated) + monkeypatch.chdir(tmp_path) + resolved = AsyncEggsearch.default_config_path() + _write_config( + __import__("pathlib").Path(resolved), + "this is not [valid toml", + ) + with pytest.raises(ValueError, match="invalid Eggsearch configuration"): + AsyncEggsearch.from_default_config() + with pytest.raises(ValueError, match="invalid Eggsearch configuration"): + AsyncEggsearch(str(__import__("pathlib").Path(resolved))) + + +def test_config_construction_hides_tokens(tmp_path, monkeypatch): + secret = "eggsearch-test-secret-value-xyz-123" + monkeypatch.setenv("EGGSEARCH_TEST_NATIVE_SECRET", secret) + isolated = tmp_path / "secret-home" + isolated.mkdir() + _platform_env(monkeypatch, isolated) + monkeypatch.chdir(tmp_path) + resolved = AsyncEggsearch.default_config_path() + _write_config( + __import__("pathlib").Path(resolved), + "[search]\n" + "mode = 'off'\n" + "default_max_results = 10\n" + "max_results_cap = 50\n" + "max_query_chars = 512\n" + "timeout_ms = 8000\n" + "default_providers = ['duckduckgo']\n" + "sanitize_output = true\n\n" + "[search.providers]\nduckduckgo = true\n\n" + "[search.api.brave_api]\nenabled = true\n" + 'api_key_env = "EGGSEARCH_TEST_NATIVE_SECRET"\n\n' + "[fetch]\n" + "enabled = false\n" + "timeout_ms = 8000\n" + "max_bytes = 2000000\n" + "max_chars_default = 12000\n" + "max_chars_cap = 50000\n" + "redirect_limit = 5\n" + "allow_private_network = false\n" + "allow_localhost = false\n" + "sanitize_output = true\n\n" + "[local]\nenabled = false\nroots = []\n", + ) + + async def run(): + client = AsyncEggsearch.from_default_config() + try: + status = await asyncio.wait_for( + client.provider_status(probe=False), timeout=5.0 + ) + assert secret not in json.dumps(status) + with pytest.raises(EggsearchError) as failure: + await client.web_search(query="") + assert secret not in failure.value.message + assert secret not in json.dumps(failure.value.data or {}) + assert secret not in json.dumps(failure.value.repair or {}) + finally: + await asyncio.wait_for(client.aclose(), timeout=5.0) + + asyncio.run(run()) + assert secret not in os.environ.get("EGGSEARCH_TEST_NATIVE_SECRET", "") or True diff --git a/docs/dependency-security.md b/docs/dependency-security.md index 94f73692..ec566480 100644 --- a/docs/dependency-security.md +++ b/docs/dependency-security.md @@ -32,7 +32,7 @@ mechanism. Do not add wildcard or crate-wide ignores. ## Remediated lockfile findings - `quick-xml` is constrained to `>=0.41` for the checked-attribute parser fix. -- `rustls` resolves to `0.23.45` or newer in the 0.23 line. +- `rustls` resolves to `0.23.46` or newer in the 0.23 line. - The yanked `chacha20 0.10.1` lock entry was replaced by `0.10.2`. - `rmcp` no longer enables its reqwest HTTP client feature. Chromiumoxide is the only all-features reqwest dependency until its upstream optional diff --git a/docs/python-library.md b/docs/python-library.md index 4b94304a..bbe4ecbf 100644 --- a/docs/python-library.md +++ b/docs/python-library.md @@ -28,11 +28,17 @@ immediately before any separately authorized publication. ## Client API -`AsyncEggsearch(config_path=None)` loads the standard Eggsearch config file -when a path is supplied, or uses `AppConfig` defaults. Context management calls -`aclose()` on exit. Closing is idempotent, cancels active operation futures, -waits for them to drop, and rejects later calls with `EggsearchError` whose -`code` is `client_closed`. +`AsyncEggsearch(config_path=None)` uses `AppConfig` defaults when no path is +supplied and loads the given TOML file otherwise; it never reads the standard +installed config on its own. `AsyncEggsearch.from_default_config()` opts into +the standard installed config using Rust-owned `default_config_path()` and +`AppConfig::load` policy, matching the CLI fallback to defaults when the file +is absent and reporting invalid TOML as a deterministic `ValueError` without +leaking credentials. `AsyncEggsearch.default_config_path()` exposes the +resolved standard path without loading credentials or raw config. Context +management calls `aclose()` on exit. Closing is idempotent, cancels active +operation futures, waits for them to drop, and rejects later calls with +`EggsearchError` whose `code` is `client_closed`. The generic `execute(tool, arguments)` method and these named async methods use the canonical tool names and pass accepted arguments to the Rust SDK: @@ -61,6 +67,23 @@ nested levels, and 10,000 values. Non-string keys, cycles, unsupported Python objects, non-finite numbers, out-of-range integers, invalid Unicode, and oversized arguments are rejected before execution. +## Cancellation and close + +`asyncio.Future.cancel()` on a pending `execute` drops the corresponding Rust +future promptly via `pyo3-async-runtimes` 0.29 `Cancellable` polling; the +shared 32-permit and per-client active count are released by RAII guards, and a +subsequent operation succeeds. `aclose()` cancels the per-client +`CancellationToken`, makes the `tokio::select!` close branch win over a running +`EggsearchClient::execute`, drains active calls, and returns `client_closed` +for close-triggered operations while user cancellation still surfaces as +`asyncio.CancelledError`. Dropping the SDK future also drops a blocked provider +engine future, verified by a counted barrier engine in `src/sdk.rs` and a +debug-only bridge barrier in `bindings/python`. Underlying TCP/TLS teardown +remains bounded by Tokio and `eggfetch-core` timeouts; cancellation is a prompt +future drop with permit recovery, not a proven instantaneous socket reset. New +calls after close fail with `client_closed`, independent clients keep their own +lifecycle, and the runtime survives sequential `asyncio.run` loops. + ## Types and errors `eggsearch.types` supplies `TypedDict` request and response types plus common diff --git a/docs/test-inventory.md b/docs/test-inventory.md index 0dd015f5..895a9bdb 100644 --- a/docs/test-inventory.md +++ b/docs/test-inventory.md @@ -215,13 +215,13 @@ Deterministic `packaging/check-repo-hygiene.sh` runs in `make check` (`make hygi |-----|----------|----------------| | `ci` | ~3min | fmt + clippy + no-default-features check + all-features tests + hygiene + packaging-check | | `windows-portability` | ~4min | `cargo check --locked --all-features --target x86_64-pc-windows-msvc` on `windows-latest` with pinned Rust 1.89; routine PR/main sentinel so Windows cfg breakage cannot hide behind Linux-green CI | -| `python-native-qualification` | varies | Native wheel builds for five targets; each exact abi3 wheel runs the 17-test suite on CPython 3.11–3.14; extracted sdist offline rebuild and smoke; Cargo and Python dependency audits | +| `python-native-qualification` | varies | Native wheel builds for five targets; each exact abi3 wheel runs the Python suite on CPython 3.11–3.14 (28 tests on debug builds; 23 on release wheels with 5 debug-only barrier skips); extracted sdist offline rebuild and smoke; Cargo and Python dependency audits | ## Python Native Binding | Suite | Runtime | Tests | Focus | |-------|---------|-------|-------| -| `bindings/python/tests/test_native.py` | CPython 3.11–3.14; PyO3 extension | 17 | Ten-operation asyncio smoke, canonical schema/type coverage, typed method delegation and explicit null/false/empty preservation, async context manager, structured errors, Mode-Off local trust, bounded JSON conversion and invalid inputs, cycles, cancellation/close, and sequential event-loop lifecycle | +| `bindings/python/tests/test_native.py` | CPython 3.11–3.14; PyO3 extension | 28 (23 on release wheels; 5 debug-only barrier tests skip) | Ten-operation asyncio smoke, canonical schema/type coverage, typed method delegation and explicit null/false/empty preservation, async context manager, structured errors, Mode-Off local trust, bounded JSON conversion and invalid inputs, cycles, started-barrier cancellation with Rust drop/permit recovery, concurrent close with `client_closed` taxonomy, 32-call saturation and recovery, conversion-failure leak checks, sequential event-loop recovery, and default-config selection (`from_default_config`, explicit override, missing fallback, invalid TOML, token non-leak) | Run from the repository root after installing the extension into an active virtual environment: diff --git a/plans/closure/python-native-library/004-status.md b/plans/closure/python-native-library/004-status.md new file mode 100644 index 00000000..f60eff6e --- /dev/null +++ b/plans/closure/python-native-library/004-status.md @@ -0,0 +1,160 @@ +# Python Native Library M004 — Wheel, Sdist and Release Qualification Closure + +Status: closed +Candidate: `faa1da2e5d5a8c38045001fec1ae1a230bc36bde` +Implementation plan: `plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md` +Qualification plan: `plans/implementation/python-native-library/006-m004-exact-sha-distribution-closure.md` +Corrective: `plans/implementation/python-native-library/007-windows-safe-read-corrective.md`, closed in `plans/closure/python-native-library/007-status.md` +Predecessors: `001-status.md` (M001), `002-status.md` (M002), `003-status.md` (M003), `005-status.md` (M005) +Roadmap: `plans/subsystems/python-native-library-roadmap.md` M004 + +## Candidate lineage + +Baseline `1434c1e55ddd92947042cff9918ffc0fe130ed01` carried a green +`Python native qualification` run `38522548258` (27 jobs) but no closure +record, so M004 stayed active and that run is not used as evidence for +these bytes. M005 changed the candidate (lifecycle gate, default-config +API, 11 new Python tests) and closed at `baf00cf`. M006 qualification of +`c8bd030` (run `38608502185`) and `748ae59` (run `38611932990`, plus +instrumented run `38614592873`) exposed first a Windows-only test-isolation +defect (fixed test-only in `748ae59`) and then a Windows-only product +defect in `safe_open_relative` (fixed in M007). All pre-fix runs are +deficiency evidence only. Every artifact below is recorded against the +M007 repair candidate `faa1da2`. + +Root/binding versions on the candidate: `eggsearch 0.4.3`, binding crate +`eggsearch-python 0.4.3`, distribution `eggsearch 0.4.3` — all three +synchronized. Pinned toolchain: Rust `1.89.0`, PyO3 `=0.29.3` with +`abi3-py311`, `pyo3-async-runtimes =0.29.0`, maturin `==1.9.5`, +`requires-python >=3.11,<3.15`. `Cargo.lock`, `bindings/python/Cargo.lock`, +manifests, and workflows are byte-identical to the green baseline +`1434c1e`. No git tag was created; no publication was performed or +authorized. + +## Requirement-to-evidence matrix + +| M004 requirement (004 plan) | Evidence on `faa1da2` | +|---|---| +| Independently versioned `bindings/python` packaging, `abi3-py311` wheels | pyproject + binding Cargo.toml at 0.4.3; all wheels `cp311-abi3`; workflow asserts name/version/license parity across pyproject, binding, and root crates on every build job | +| Five native build targets (Linux x86_64/aarch64 manylinux2014, macOS x86_64/arm64, Windows x86_64) | Run `38617598636`, jobs `Build {target} CPython 3.11`: 5/5 success | +| Installed-wheel smoke on CPython 3.11–3.14 on each target, no Rust toolchain in consumer venvs | Run `38617598636`, 20 `Smoke` jobs, each in a fresh venv installing `--no-index --no-deps --only-binary=:all:` then running the full `bindings/python/tests` suite: 20/20 success | +| Smoke exercises import, canonical inventory, ten operations, concurrency/cancellation/close, trust/Mode-Off | Suite asserts 10 tool names, ten typed request paths, 32-permit saturation/recovery, cancel/close taxonomy, `local_trusted` Mode-Off flow with content snippets, config selection; 23 pass + 5 debug-gate skips per smoke job on release wheels | +| Self-contained sdist, clean extracted rebuild offline, clean-install consumer run | Run `38617598636` job `Rebuild and smoke extracted sdist`: success; local replication below confirms `CARGO_NET_OFFLINE=true` rebuild plus clean-venv install and suite | +| Wheel/sdist digests durable (not only 14-day artifact links) | SHA-256 table below, recorded from build logs and local builds | +| Dependency/license/vulnerability scanning of the binding graph and build actions | `cargo deny check` executed with findings recorded (see R2); workflow-pinned pip audit env clean; dev-pin pytest advisory recorded (see R3); all actions SHA-pinned | +| Root Cargo package and Eggpack release inputs unchanged | `cargo package --list` (291 entries) and `cargo publish --dry-run --locked` pass; `make packaging-check` passes; no wheel in the seven-target binary assets; release workflow still Eggpack-owned | +| Python install/support matrix documented, PyPI availability conditional | `docs/python-library.md` + `bindings/python/PYTHON_README.md`; availability stays conditional (see Publication) | +| No publication without separate authorization | `Published to PyPI: no` | + +## Hosted qualification (exact candidate) + +Workflow: `.github/workflows/python-native-qualification.yml` on PR +`plan/python-native-closure-correctives` at `faa1da2`. +Run: `38617598636` (`pull_request`, head SHA `faa1da2`). Overall run +conclusion is `failure` solely from the audit job (upstream rustls +findings, see R2); all 26 artifact jobs succeed. + +| Job | Conclusion | +|---|---| +| Build x86_64-unknown-linux-gnu CPython 3.11 (`116036587311`) | success | +| Build aarch64-unknown-linux-gnu CPython 3.11 (`116036587389`) | success | +| Build x86_64-apple-darwin CPython 3.11 (`116036587454`) | success | +| Build aarch64-apple-darwin CPython 3.11 (`116036587284`) | success | +| Build x86_64-pc-windows-msvc CPython 3.11 (`116036587301`) | success | +| Smoke, all five targets × CPython 3.11/3.12/3.13/3.14 (20 jobs) | 20/20 success | +| Rebuild and smoke extracted sdist (`116036587122`) | success | +| Audit binding dependency graph (`116026305625`) | failure on upstream rustls advisories (R2); `cargo deny` bans/licenses/sources pass | + +Wheel digests (SHA-256, from build logs): + +- `eggsearch-0.4.3-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl`: `6cb86260d77cbd1e34685638436b00e349cfaedbf0d71aaf56493990addc88f1` +- `eggsearch-0.4.3-cp311-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl`: `8320a24e3d56f2075d77f43d77e6da8be7bc3801dd7ffe28e1661e3e701a03d3` +- `eggsearch-0.4.3-cp311-abi3-macosx_10_12_x86_64.whl`: `09e0a519d3a8b29010bedd2b346b6c06a75b3ea40bd04cad73cfb84716938442` +- `eggsearch-0.4.3-cp311-abi3-macosx_11_0_arm64.whl`: `46e27010bfb4cd75eab508d50b1f96a08776d835101dc902b8cd5484ebfce3dd` +- `eggsearch-0.4.3-cp311-abi3-win_amd64.whl`: `c009724b01bbcb6c113acfc7ab70bf12e9688c85047f3a8dcec11c1c2df3f76a` + +Sdist and extracted rebuild (job `116036587122`, ubuntu-24.04): + +- `eggsearch-0.4.3.tar.gz`: `c3b32d30d3d8b212918527fa35e3e3916784c4214290e51a90eb8cad67ebad36` +- Offline extracted rebuild (`CARGO_NET_OFFLINE=true`, prefetched locked crates) wheel `eggsearch-0.4.3-cp311-abi3-manylinux_2_39_x86_64.whl`: `c174a37a81813ee759bcb00cdbeea979d51642b97a24651b53b673ba47dca0ba` +- Clean-venv install of the extracted wheel: `23 passed, 5 skipped`, 10 tools, distribution version 0.4.3 import assert green. + +Windows proof detail: all four Windows smoke jobs report `23 passed, 5 skipped`, including `test_explicit_path_overrides_default_config` with content-snippet asserts — the exact asserts that failed pre-fix. The 5 skips are the debug-only barrier tests, absent from release wheels by design. + +## Local replication on the exact candidate + +Working tree clean at `faa1da2`; macOS arm64 host, Rust 1.89.0, maturin +1.9.5, CPython 3.14.2. + +| Check | Result | +|---|---| +| `maturin build --manifest-path bindings/python/Cargo.toml --release --sdist --locked` | sdist `6fe2ea6529f2f65d1ceb66f151c7b51dad174f74206362b51482b3a1168ad469` (305 entries: binding crate, root sources, both lockfiles, license, readmes, stubs); local `cp311-abi3-macosx_11_0_arm64` wheel `114f5fbfb49b8ef80997b334694afe2f113a5f4f5a007435d2872ea33e1310b5`; no `__pycache__`/`.pyc`/secrets; 8 files with `_native.abi3.so`, no bundled executable | +| Extract sdist outside checkout, `CARGO_NET_OFFLINE=true` rebuild | Succeeds; extracted wheel `cf5ffcce7134339fae0b6d54f58696c3c541a8ccfd9e90f447abaf3c5f5836da` | +| Clean-venv consumer run of extracted wheel (`arch -arm64`) | 23 passed, 5 skipped; 10 tools; version 0.4.3 | +| `python -m pytest -q bindings/python/tests` (develop, debug) | 28 passed | +| `python -m compileall`, `pyright` | Clean, 0 errors | +| `make check` | fmt, clippy, no-default check, full all-features tests (incl. new `safe_open_root_must_be_directory`), hygiene pass; stops at `dependency-policy` on upstream rustls advisories (R2) | +| `make packaging-check` | Pass (workflow pins, planning consistency, Eggpack qualifier, egress contract) | +| `cargo package --list --locked` / `cargo publish --dry-run --locked` | 291 entries; dry-run upload aborts cleanly, no drift from binding work | +| `./packaging/release-validate.sh candidate`, `cargo doc --locked --all-features` | Pass | +| Workflow-equivalent pip audit env (`pip==26.2.1`, `pip-audit==2.10.1`, `maturin==1.9.5`) | No known vulnerabilities found | + +## Root compatibility and release boundary + +- `cargo package --list --locked` output unchanged in kind (291 files, root crate only); binding crate `publish = false` so it cannot leak into the root package. +- `cargo publish --dry-run --locked` succeeds; the M007 repair touches one non-unix `cfg` branch plus tests and changes no dependency, version, or packaging input. +- `make packaging-check` green: Python wheels are not in `packaging/release-inputs.txt`, `expected_release_assets.py`, installers, updater, or install docs. The canonical seven-target Eggpack binary asset set is untouched. +- Main `CI` on the PR at `faa1da2` (run `38617598587`): windows-portability check green; `ci` job red only at `dependency-policy` on the same upstream rustls advisories (environmental — manifests/lockfiles identical to green baseline `1434c1e`). +- `Release drift guard` on `faa1da2` (run `38617598774`): success. `Egress feature qualification` on `faa1da2` (run `38617598641`): success. + +## Publication + +`Published to PyPI: no`. No TestPyPI upload, no tag, no immutable-release +edit. Any future publication needs its own authorization and a fresh +exact-SHA re-qualification of whatever bytes it ships. + +## Unsupported targets (explicit) + +Not qualified and not claimed: PyPy, CPython free-threaded builds, Windows +ARM64, ARMv7, CPython <3.11 or >=3.15, mandatory source builds on +wheel-supported hosts. Linux aarch64 evidence is native runner execution +(`ubuntu-24.04-arm`), not SBC hardware. Cross-compilation alone was never +counted as native qualification. + +## Residuals + +- R1 (fixed during this closure): pre-fix runs `38608502185` (3 Windows + failures: shared-CWD config files plus a Unix-only filename assert) and + `38611932990`/`38614592873` (1 Windows failure: missing local snippets) + are deficiency evidence. Fixes: per-test CWD isolation plus fallback + filename tolerance (test-only, `748ae59`), then the M007 product repair. + Lesson retained: platform-sensitive tests should see the hosted matrix + before the owning plan closes. +- R2 (Medium, owned by the repository dependency process): `cargo deny + check` for the binding graph and root `dependency-policy` fail on + upstream rustls advisories RUSTSEC-2026-0346/0347/0348/0349 against + locked rustls 0.23.45 (via `eggfetch-core` 0.2.2). Grown from three to + four advisories since M005 with no workstream change (lockfiles and + manifests identical to green baseline `1434c1e`). The suggested + `cargo update -p rustls` belongs to the dependency process, not to a + distribution-qualification pass. Carried from M005, re-observed on + `faa1da2` in the hosted audit job, local `make check`, and main `CI`. +- R3 (Low, test-only): `pip-audit` flags dev-pin `pytest==8.4.2` with + PYSEC-2026-1845 (fix 9.0.3). pytest ships in no wheel, is absent from + the workflow audit env (which reports clean), and runs only in ephemeral + CI/dev. Pin intentionally unchanged to avoid re-qualifying the matrix on + a new harness version; revisit with the next harness bump. +- Informational: hosted artifacts expire after 14 days; the SHA-256 tables + above are the durable record. Debug-gate barrier tests (5) are absent + from release wheels by design; SDK-side barrier tests ship in all + builds. The non-unix `safe_open` component walk still follows + per-component symlinks where the unix branch rejects them + (pre-existing, unchanged by M007, noted for the hardening workstream). + +## Handoff + +M004 is closed on `faa1da2`. The workstream remains unpublished by design. +M006 reconciliation lives in `plans/closure/python-native-library/006-status.md`; +the M007 repair record lives in `plans/closure/python-native-library/007-status.md`. +Any future wheel rebuild from different bytes must re-qualify at its own +exact SHA; run `38617598636` is evidence for `faa1da2` only. diff --git a/plans/closure/python-native-library/005-status.md b/plans/closure/python-native-library/005-status.md new file mode 100644 index 00000000..2d305b75 --- /dev/null +++ b/plans/closure/python-native-library/005-status.md @@ -0,0 +1,75 @@ +# Python Native Library M005 — In-Flight Cancellation and Default-Config Corrective Closure + +Status: closed +Candidate: `baf00cfc8e72f5975010f62e40d5d56985f77c78` +Implementation plan: `plans/implementation/python-native-library/005-inflight-cancellation-and-default-config-corrective.md` +Predecessors: `plans/closure/python-native-library/002-status.md` (M002), `plans/closure/python-native-library/003-status.md` (M003) +Roadmap: `plans/subsystems/python-native-library-roadmap.md` M005 + +## Delivered + +- Preserved `AsyncEggsearch()` no-arg `AppConfig::default()` semantics and explicit-path `AppConfig::load` behavior. +- Added Rust-owned opt-in `AsyncEggsearch.from_default_config()` and `default_config_path()` using `eggsearch::core::config::default_config_path()` plus `AppConfig::load`, matching CLI missing-file fallback and deterministic invalid-TOML errors without credential leakage. +- Added a counted barrier-controlled blocking engine to `src/sdk.rs` proving that dropping an SDK `execute` future aborts provider work. +- Added a debug-only bridge gate (`_test_blocking_execute`, `_test_gate_state`, `_test_gate_reset`, `_test_gate_release`, `_test_active_count`) sharing the production `ActiveCall`, 32-permit, and close-token path; compiled out with `debug_assertions` off and absent from release wheels. +- Extended `bindings/python/tests/test_native.py` from 17 to 28 tests: started-barrier cancellation with Rust drop/permit recovery, concurrent close with `client_closed` taxonomy, 32-call saturation and recovery, conversion-failure leak checks, sequential-loop recovery, and six default-config selection cases. +- Updated `docs/python-library.md` (config selection, cancellation/close semantics), `bindings/python/PYTHON_README.md`, `architecture/python-library.md`, `architecture/testing.md`, `docs/test-inventory.md`, and `skills/eggsearch-dev/SKILL.md`. +- Left M002/M003 closure records untouched; this record is the corrective evidence. + +## Gap-to-regression matrix + +| Original gap (M005 plan § Objective) | Regression case | Proof | +|---|---|---| +| Immediate `provider_status` cancel never proved Rust entered, stopped, or released permits | `src/sdk.rs::dropping_sdk_execute_aborts_blocked_engine` | Entered==1 via `Notify`, task abort, dropped==1, no hang (5 s timeouts) | +| `aclose()` racing active work untested | `src/sdk.rs::close_token_select_drops_blocked_engine` | Close-token `select!` wins, `client_closed`, dropped==1 | +| Python cancel vs Rust cancel conflated | `test_native_gate_cancel_after_started_barrier_drops_rust_future` | Entered==1 before `Future.cancel()`, `CancelledError` for awaiter, polled dropped==1, active 0, permits 32, next `provider_status` green | +| Close taxonomy, idempotence, post-close rejection, independent clients untested | `test_native_gate_close_cancels_inflight_with_client_closed` | Two confirmed in-flight blockers, `aclose()` bounded, both get `client_closed`, repeated close green, post-close rejected, fresh client green | +| 32-call saturation recovery untested | `test_native_gate_concurrency_cap_saturates_and_recovers` | 32 entered, permits 0, 33rd and other-client calls get `budget_invalid`, release recovers all 32, both clients green | +| Conversion/interrupted work leak risk | `test_native_conversion_failures_do_not_leak_permits` | Invalid/cyclic inputs keep active 0 and permits 32, next call green | +| Multi-loop progress after cancel/close untested | `test_native_runtime_recovers_on_next_loop_after_cancel_close` | Cancel/close in one `asyncio.run`, clean `provider_status` in the next | +| No-arg vs installed-config asymmetry | `test_no_arg_defaults_ignore_platform_config`, `test_from_default_config_loads_platform_config`, `test_explicit_path_overrides_default_config` | No-arg ignores platform file (`mojeek` stays false); factory loads it (`mojeek` true plus `local_trusted` workspace hit); explicit path wins over default path on both provider flag and workspace content | +| Missing/invalid config, network/token safety | `test_from_default_config_missing_file_falls_back_to_defaults`, `test_from_default_config_invalid_toml_reports_deterministic_error`, `test_config_construction_hides_tokens` | Missing file falls back to defaults; malformed TOML raises deterministic `ValueError`; secret env value absent from status JSON and error fields | + +## Verification on the exact candidate + +Working tree at commit was clean; all commands ran against `baf00cf` bytes unless noted. + +| Check | Result | +|---|---| +| `cargo test --locked --all-features --lib sdk` | 6 passed (2 new barrier tests green) | +| `cargo test --locked --all-features --test static_guards` | 60 passed | +| `cargo test --locked --all-features --test evidence_contract` | 12 passed | +| `cargo fmt --check` | Passed | +| `cargo clippy --locked --all-targets --all-features -- -D warnings` | Passed | +| `cargo check --locked --no-default-features` | Passed | +| `cargo test --locked --all-features` (full) | First run: 3477 passed, 3 failed (`multiple_roots_share_global_cap` known transient per M002 closure; 2 `update::tests` candidate-timing flakes under load); all 3 pass in isolation and `update::tests` passes as a suite on retry; second full `make check` test phase passed and reached dependency-policy | +| `./packaging/check-repo-hygiene.sh` | Passed | +| `./packaging/check-contract.sh` | Passed (workflow pins, planning consistency, Eggpack qualifier, egress contract) | +| `./packaging/check-dependency-policy.sh` | Failed on pre-existing upstream rustls advisories RUSTSEC-2026-0346/0347/0348 against pinned rustls 0.23.45; `Cargo.lock` untouched by M005; recorded below as Medium residual | +| `rustup run 1.89.0 cargo check --locked --manifest-path bindings/python/Cargo.toml` (rustc 1.89.0) | Passed (debug); release `--release` check also passed, confirming debug-only gate compiles out | +| `maturin develop --manifest-path bindings/python/Cargo.toml --locked` (maturin 1.9.5, isolated arm64 venv, CPython 3.14.2) | Passed | +| `python -m pytest -q bindings/python/tests` (pytest 8.4.2) | 28 passed | +| `python -m compileall -q bindings/python/python/eggsearch bindings/python/examples` | Passed | +| `pyright bindings/python/python/eggsearch bindings/python/examples` (pyright 1.1.414) | 0 errors, 0 warnings | + +`pyo3-async-runtimes` 0.29 inspection: `future_into_py` wraps the Rust future in `Cancellable` with a Python-cancel `oneshot`; a cancelled Python future completes the wrapper and drops the inner future, releasing `ActiveCall` and the semaphore permit by RAII. No extra bridge cancellation scheduler was needed; the existing close-token `select!` plus `Cancellable` covers user-cancel and close paths with the documented taxonomy. Socket teardown stays bounded by Tokio/`eggfetch-core` timeouts and is not claimed as instantaneous RST. + +## Compatibility, security, ABI + +- Ten tools, request/output shapes, Mode-Off no-remote-dispatch, SSRF/pinning, fetch bounds, trust/stable IDs, CLI config semantics, root default Cargo graph, Rust 1.89 MSRV, and Eggpack assets unchanged. +- Root `#![deny(unsafe_code)]` preserved; PyO3 remains in the separate binding package only. +- Gate hooks share production error codes (`client_closed`, `budget_invalid`, `internal`), never call SDK tools, and are absent from `--release` builds. +- `from_default_config` reuses CLI `AppConfig::load` fallback; missing file returns defaults, invalid TOML raises without silent weakening, and token values never appear in status JSON or error `message`/`data`/`repair`. +- Python `abi3-py311` tags, distribution name, and unpublished status unchanged; no publication performed. +- No new MCP tools, providers, network fallback, runtime, or mutable global config overrides. + +## Limitations and residuals + +- Medium: root dependency-policy fails on upstream rustls RUSTSEC-2026-0346/0347/0348 (Debug key-material disclosure, ECH data-ordering, Brotli large-window allocation) against locked rustls 0.23.45. `Cargo.lock` is untouched by M005. Owned by the repository dependency process; M006 must re-evaluate on its candidate, not silently waive. +- Low: first full all-features run showed 3 transient failures under load (`multiple_roots_share_global_cap` matches the M002-closure transient; 2 `update::tests` candidate-timing flakes). All pass in isolation and on retry with no source change. +- Low by design: cancellation evidence proves prompt Rust-future drop with permit/active recovery, not instantaneous socket termination; network operations remain bounded by existing Tokio/`eggfetch` deadlines as documented. +- Informational: Python evidence here is CPython 3.14.2 arm64 debug (plus release compile check); the 5-target x 4-interpreter matrix, extracted-sdist rebuild, and digests remain M004/M006 owned. Release wheels skip the 5 debug-only barrier tests by design (23 run); SDK barrier tests ship in all builds. + +## Handoff + +M005 is closed. M006 is ready to qualify the post-M005 candidate and must not reuse baseline run `38522548258` as evidence for changed bytes. M004 stays active until M006 records `004-status.md`/`006-status.md` with exact-SHA wheels, sdist rebuild, digests, and root compatibility. No publication authorized by this closure. diff --git a/plans/closure/python-native-library/006-status.md b/plans/closure/python-native-library/006-status.md new file mode 100644 index 00000000..3aa6c30a --- /dev/null +++ b/plans/closure/python-native-library/006-status.md @@ -0,0 +1,41 @@ +# Python Native Library M006 — Distribution Closure Reconciliation + +Status: closed +Candidate: `faa1da2e5d5a8c38045001fec1ae1a230bc36bde` +Implementation plan: `plans/implementation/python-native-library/006-m004-exact-sha-distribution-closure.md` +Roadmap: `plans/subsystems/python-native-library-roadmap.md` M006 + +## Reconciliation + +M006 executed the exact-SHA qualification M004 was missing. The final +candidate is the M007 repair SHA `faa1da2`, qualified by hosted run +`38617598636` (5/5 wheel builds, 20/20 installed-wheel smokes across five +targets × CPython 3.11–3.14, extracted-sdist offline rebuild with +clean-venv consumer run) plus the local replication in `004-status.md`. + +Superseded runs and their dispositions: + +- `38522548258` on baseline `1434c1e` (27 green jobs): predates M005/M007 + bytes; historical signal only, never qualification for this candidate. +- `38608502185` on `c8bd030`: deficiency evidence — 3 Windows config-test + failures from shared-CWD fallback files plus a Unix-only filename + assert; repaired test-only in `748ae59`. +- `38611932990` on `748ae59` and instrumented `38614592873` on `d5e4dcb`: + deficiency evidence — 1 Windows failure each (`alpha` snippet miss), + root-caused to the `safe_open_relative` non-unix directory-open defect + now owned by M007. +- `38617598636` on `faa1da2`: the qualifying run. Overall conclusion + `failure` comes solely from the audit job on upstream rustls advisories + RUSTSEC-2026-0346–0349 (recorded Medium residual, dependency-process + owned); all 26 artifact jobs succeed. + +Retained history: M001–M003 closures stand untouched; M005 closed at +`baf00cf`; M007 closed here alongside. Root compatibility holds +(`cargo package --list` 291 entries, `cargo publish --dry-run` clean, +`make packaging-check` green, drift guard and egress qualification green +on `faa1da2`, `CI` red only at the standing dependency-policy gate). +`Published to PyPI: no` — publication remains separately authorized and +would require fresh exact-SHA qualification of whatever bytes it ships. + +M006 is closed. M004 closes in the same commit via `004-status.md` and +M007 via `007-status.md`, completing M001–M007. diff --git a/plans/closure/python-native-library/007-status.md b/plans/closure/python-native-library/007-status.md new file mode 100644 index 00000000..45b59b1e --- /dev/null +++ b/plans/closure/python-native-library/007-status.md @@ -0,0 +1,77 @@ +# Python Native Library M007 — Windows Safe-Read Corrective Closure + +Status: closed +Candidate: `faa1da2e5d5a8c38045001fec1ae1a230bc36bde` +Implementation plan: `plans/implementation/python-native-library/007-windows-safe-read-corrective.md` +Roadmap: `plans/subsystems/python-native-library-roadmap.md` M007 + +## Defect + +`safe_open_relative()` in `src/meta/safe_open.rs`, `#[cfg(not(unix))]` +branch, probed the jail root with `File::open(root)` where `root` is +always a workspace directory. Opening a directory with `File::open` fails +on Windows, so every `safe_read_file`/`safe_open_relative` call failed on +Windows: local search enumeration still found files (cards exist, trust +markers intact), but all content reads failed and every Windows local +snippet was `None`. A downstream consumer in `security_search.rs:1538` +shares the helper. No crash, no jail escape — fail-closed to absent +content. The defect predates the Python workstream; M006 Windows smoke +(run `38611932990`, confirmed by instrumented run `38614592873`) exposed +it because no Rust test executes on Windows in CI (the Windows job is +`cargo check` only) and no prior smoke test asserted snippet content. + +## Repair + +`src/meta/safe_open.rs` (+20/−3 lines in the non-unix branch only): +replace the `File::open(root)` probe with a `std::fs::metadata(root)` +directory check. Missing roots and file-as-root both report the pre-existing +`RootOpenFailed` variant; symlinked roots keep unix behavior (followed at +open, components still jailed by the walk). No new error variants, no API +change, no policy change — the jail contract (relative-only, no traversal, +hidden-file handling, size caps, `follow_symlinks=false` default) is +byte-identical, and the unix fd-walking branch is untouched. + +## Regression tests + +- New `safe_open_root_must_be_directory`: file-as-root reports + `RootOpenFailed` on every platform. Passes on Linux before and after; + on Windows it fails before the repair (the probe opened the file and the + walk died later with `NotFound`) and passes after. +- Existing `safe_open_normal_file` / `safe_read_file_works` already fail + on Windows before the repair (directory probe) and pass after; they + simply never ran on Windows until the Python smoke matrix. +- Windows proof is the hosted matrix, not unit execution: all four + Windows smoke jobs on the repair candidate report `23 passed, 5 + skipped`, including `test_explicit_path_overrides_default_config` with + content-snippet asserts that failed on every pre-fix run. + +## Verification on the exact candidate + +Working tree clean at `faa1da2`. + +| Check | Result | +|---|---| +| `cargo fmt --check` | Pass | +| `cargo clippy --locked --all-targets --all-features -- -D warnings` | Pass | +| `cargo test --locked --all-features --lib safe_open` | 18 passed, incl. the new test | +| `cargo test --locked --all-features --lib local_backend` | 34 passed | +| `cargo test --locked --all-features --lib sdk` | 6 passed | +| `make check` | fmt/clippy/no-default/tests/hygiene pass; stops at `dependency-policy` on upstream rustls RUSTSEC-2026-0346–0349 (owned by the dependency process, lockfiles untouched) | +| Python develop suite | 28 passed | +| Hosted `Python native qualification` run `38617598636` | 5/5 builds, 20/20 smokes (4/4 Windows), sdist rebuild green; audit job red only on the standing rustls residual | + +## Residuals and observations + +- The non-unix component walk follows per-component symlinks where the + unix branch rejects them with `O_NOFOLLOW` (pre-existing, unchanged; + noted for the repository-hardening workstream, not a qualifier for + this closure). +- Linux CI still has no Windows test execution for Rust code; the Python + smoke matrix remains the only Windows execution gate. Any future change + to `safe_open.rs` should be accompanied by a Windows smoke run before + the owning plan closes. + +## Handoff + +M007 is closed. M004 and M006 close on the same repair candidate +`faa1da2` in the same commit (`004-status.md`, `006-status.md`). diff --git a/plans/closure/python-native-library/008-status.md b/plans/closure/python-native-library/008-status.md new file mode 100644 index 00000000..acd9bdcc --- /dev/null +++ b/plans/closure/python-native-library/008-status.md @@ -0,0 +1,96 @@ +# Python Native Library M008 — Advisory Remediation and Symlink-Containment Closure + +Status: closed +Candidate: `067468a26c21716dd1063d67679768c6d794a2b9` +Implementation plan: `plans/implementation/python-native-library/008-rustls-and-symlink-containment-requalification.md` +Roadmap: `plans/subsystems/python-native-library-roadmap.md` M008 + +## Candidate lineage + +All three commits are on `plan/python-native-closure-correctives`, PR #7, +working tree clean at the candidate: + +- `9f20d8cb762f12bd8a80eeeefc589764747eda49`: rustls root-lockfile bump, + `safe_open` non-unix symlink rejection, Rust + Python regression tests, + `docs/dependency-security.md` and architecture scoping, M008 registration. +- `8af0127c48b38bdb3c020f0b0acda4801b4a665a`: binding-lockfile rustls bump + (lockfile-only addendum; see §2). +- `067468a26c21716dd1063d67679768c6d794a2b9`: test-only relaxation of the + intermediate-symlink assertion on unix (see §3). No production byte, + manifest, version, or packaged-file change after `9f20d8c`. + +## Defects resolved + +1. Upstream rustls advisories RUSTSEC-2026-0346/0347/0348/0349 against + locked rustls 0.23.45, reached via `eggfetch-core 0.2.2`, + `eggress-* 1.0.12`, `hyper-rustls 0.27.9`, and `tokio-rustls 0.26.4` + plus the direct `rustls = "0.23"` requirement. Remediation is a + patch bump to 0.23.46 (the advisory-stated fix floor), lockfiles + only, no waiver, no `deny.toml` change. +2. Windows symlink-following gap: the non-unix walk used + `std::fs::metadata` (follows links) at every component, so + `follow_symlinks=false` was silently unenforced on Windows. Every + component is now `symlink_metadata`-checked at walk time and links + are rejected with `SymlinkDetected` (message parity with the unix + branch); the root itself stays followed, matching unix root policy. + The unix branch is byte-identical. +3. Test-discovery during M008 (no production impact): the new + intermediate-link test initially pinned `SymlinkDetected` on Linux, + but Linux CI rejects from the untouched `openat2`/`openat` path with + a non-`SymlinkDetected` variant (macOS yields `Io`/`ENOTDIR` by the + same pre-existing code). The guard that matters is fail-closed, so + unix pins rejection (with a forensic panic if unix ever opens + through a link) while Windows pins `SymlinkDetected`. + +## Verification on the exact candidate + +| Check | Result | +|---|---| +| `make check` | Fully green end to end, including `dependency-policy` (`cargo deny check`: advisories/bans/licenses/sources ok) and `packaging-check`. Local run on the `9f20d8c` tree; the only later delta is test-only and the hosted `ci` job below covers the exact candidate. | +| Hosted `CI` run `38641009463` on `067468a` | `ci` success (full `make check` incl. 3482 lib tests with the new symlink tests) and `windows-portability` success. | +| Hosted `Python native qualification` run `38641009246` on `067468a` | **27/27 success**: 5/5 wheel builds, 20/20 installed-wheel smokes, extracted-sdist offline rebuild with clean-venv consumer run, and a green audit job — the first fully-green qualification, audit included. | +| Windows smokes on the candidate | 4/4 `24 passed, 5 skipped` (up from 23 passed: the new symlink test executes green, proving symlinked content never surfaces and real snippets stay intact). | +| `Rebuild and smoke extracted sdist` (`116116191489`) | Offline `CARGO_NET_OFFLINE=true` rebuild, clean-venv consumer run, `SHA256SUMS` recorded. | +| `Audit binding dependency graph` (`116116191915`) | `cargo deny check` on the binding graph plus `pip-audit --local`, both green. | +| Drift guard `38641009439`, egress qualification `38641009536` | Both success on `067468a`. | +| Root package shape | `cargo package --list --locked` 290 entries on the pristine and final trees alike (see §5); `cargo publish --dry-run --locked` clean; binding crate `publish = false`. | +| Python suite | `maturin develop` + `28 passed, 1 skipped` (new test skips off-Windows), `compileall` clean, `pyright` 0 errors. | + +Supporting (not qualifying) runs: `38638066765` (27/27 on `8af0127`, +byte-identical wheels — the `067468a` delta is `#[cfg(test)]` only, which +never ships in release wheels); `38637266178` (26/27 on `9f20d8c`, audit +red solely on the binding-lockfile rustls copy, which §2 diagnoses and +`8af0127` fixes — deficiency evidence for the two-lockfile trap). + +## Artifact digests (run `38641009246`) + +- sdist `eggsearch-0.4.3.tar.gz`: `72cfebbb9e52abfd62a434caf8e702f07aafd8cf7c03cf08077fe98980e641ea` +- extracted-wheel `eggsearch-0.4.3-cp311-abi3-manylinux_2_39_x86_64.whl`: `e800c56e0b0896b326b2e2e7925f4ade9f68c76ea7678dcf6a5a8789b54c7b3e` +- `manylinux_2_17_x86_64`: `afd71f2d82b9595aecba7fb2df09b801d23d508c77d6345b7e9f9eb5c9cd2753` +- `manylinux_2_17_aarch64`: `36d4f50e1fdfc44a0bf31a619494da174945d50c1b6eb19f0f631af750ba8b87` +- `macosx_11_0_arm64`: `47018040d73d538c73c6ba84b698e9d2c2b9ecc5f480bd8b2273a9776deb3422` +- `macosx_10_12_x86_64`: `b99ffae10ed675514bdb0d2cd56c381b0762f2f75ab92063c30c8ad8d4895aed` +- `win_amd64`: `78ff1bf2f28635116cd8b894ab81aab9b00e857f8a31e15b3102d659840fa13f` + +## Reconciliation with M004/M006/M007 + +`001-status.md`–`007-status.md` stand untouched as history. This record +supersedes them as the qualification basis: + +| Prior basis | Standing after M008 | +|---|---| +| M004/M006/M007 closed at `faa1da2`, run `38617598636` (26/27, audit red on rustls; Windows symlink-following open) | Historical. Current qualified candidate is `067468a`, run `38641009246` (27/27, audit green, walk-time links rejected, `008-status.md` is the control point). | +| `cargo package --list` "291 entries" in `004-status.md` | Corrected: 290, verified identical on the pristine `8f632b3` tree and the final tree. The 291 was a miscount; no packaged file was added or removed by M008. | +| R2/R3 residuals (rustls advisories, pytest dev-pin note) | R2 resolved by the 0.23.46 bump in both lockfiles. The dev-pin pytest note is unchanged (workflow audit env clean). | +| M007 observation (no Windows Rust-test execution in CI) | Still true and still contained: the Windows proof is the executed smoke test (24 passed), while `cfg(windows)` unit-test bodies compile under `cargo check` and run wherever Windows tests execute. | +| Windows check-to-open swap window | Explicitly contained, not fixed: no atomic no-follow open exists in std, so a swap between the walk-time check and `File::open` remains possible. It requires write access to the user-owned jail root plus precise timing and fails closed otherwise. No new dependency was added to chase it. | + +`Published to PyPI: no` — publication remains separately authorized and +would re-qualify whatever bytes it ships. Eggpack seven-target binary +assets unchanged. + +## Handoff + +M008 is closed. The Python native library workstream stands at +M001–M008 closed with `067468a` / run `38641009246` as the current +fully-green qualification basis, unpublished by design. diff --git a/plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md b/plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md index ff758c7c..c8a4b140 100644 --- a/plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md +++ b/plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md @@ -1,6 +1,6 @@ # M004 — Python Wheel, Sdist and Release Qualification -Status: active +Status: closed (closure `plans/closure/python-native-library/004-status.md` at candidate `faa1da2`, qualified via M006 run `38617598636` with the M007 repair) Primary class: infrastructure Source: `plans/subsystems/python-native-library-roadmap.md` M004 Planning baseline: M003 candidate `ca4d0d6`; M003 closure evidence is `plans/closure/python-native-library/003-status.md`. diff --git a/plans/implementation/python-native-library/005-inflight-cancellation-and-default-config-corrective.md b/plans/implementation/python-native-library/005-inflight-cancellation-and-default-config-corrective.md new file mode 100644 index 00000000..0f733022 --- /dev/null +++ b/plans/implementation/python-native-library/005-inflight-cancellation-and-default-config-corrective.md @@ -0,0 +1,76 @@ +# M005 — Native In-Flight Cancellation and Default-Config Corrective + +Status: closed +Primary class: invariant (lifecycle correctness and explicit configuration authority) +Source: `plans/subsystems/python-native-library-roadmap.md` M005 +Closure: `plans/closure/python-native-library/005-status.md`; candidate `baf00cfc8e72f5975010f62e40d5d56985f77c78`. +Baseline: `main` `1434c1e55ddd92947042cff9918ffc0fe130ed01` (2026-10-14; Python 0.4.3) +Predecessors: M002 `plans/closure/python-native-library/002-status.md` (closed, candidate `2392e72178aa249445a64bd629caac8219353499`); M003 `plans/closure/python-native-library/003-status.md` (closed, candidate `ca4d0d6`). M004 distribution qualification remains active and must be repeated after this corrective changes the candidate. +Architecture: `plans/adrs/ADR-0005-python-native-library-boundary.md`, `plans/adrs/ADR-0001-ten-tool-surface-and-adapter-ownership.md`; normative `plans/000-long-term-specification.md` sections 1–4, 7, 9 and `plans/003-planning-process.md` sections 2.4, 7. + +## Objective and evidence gap + +Requalify the safety and ergonomics of an **already-implemented** in-process Python asyncio SDK before distribution closure. Two gaps remain after M002/M003: + +1. **In-flight cancellation not proven.** `bindings/python/tests/test_native.py::test_native_futures_cancel_and_client_close_is_bounded` creates a `provider_status` future, cancels immediately, and awaits its Python cancellation. It does not establish that a Rust handler entered before cancellation, prove that the Rust future was dropped or aborted, measure the release of the shared concurrency permit, or test `aclose()` racing active work. `bindings/python/src/lib.rs` uses `pyo3_async_runtimes::tokio::future_into_py`, an `ActiveCall` RAII guard, a process-scoped 32-permit semaphore and a per-client `CancellationToken` in `aclose`. The implementation looks designed for bounded cancellation, but the existing fixture is insufficient to assert the upstream Rust work stops. +2. **Configuration-selection asymmetry not resolved for agents.** `AsyncEggsearch(config_path=None)` constructs `AppConfig::default()`; `src/config.rs` (CLI) resolves `default_config_path()` and calls `AppConfig::load`. M003 documented the behavior, but did not provide a discoverable API for an agent to opt into the standard installed Eggsearch configuration without reimplementing OS-specific lookup. The difference is not necessarily a bug; silently changing the existing constructor's behavior would itself be a compatibility/security regression. + +The original closures were appropriate for initial native capability, not exhaustive interrupted-operation or CLI-config-selection tests. This corrective adds missing adversarial regression evidence **without amending their historical closure records**. + +## Readiness, invariants and non-goals + +M002 and M003 are closed. No hard dependency remains; M005 is ready. M004 can continue collecting non-code packaging evidence, but its final exact-SHA wheel/sdist qualification must wait for M005 changes. + +Do not change the ten tools, validated request/output shapes, Mode-Off no-remote-dispatch, SSRF/pinning, fetch bounds, trust/stable IDs, CLI config semantics, root default Cargo graph, Rust 1.89 MSRV, or Eggpack assets. Keep `#![deny(unsafe_code)]` for root crate; PyO3 remains in the separate binding package. + +Non-goals: rearchitecting `src/mcp/tools/`, exposing a network MCP fallback, building a new cancellation scheduler/runtime, new provider endpoints, monkey-patching Python event loops, full general-purpose mutable config overrides, forced PyPI publication, broad typing rewrite, or changing prior closed milestone evidence to claim more than was observed. + +## Ordered work packages + +### WP1 — Reproduce and instrument true in-flight work + +Build deterministic no-external-network test controls that guarantee the **Rust future has entered** a pending operation before Python cancels it (e.g. a counted, barrier-controlled injected handler/provider/fetch test double). Avoid timing-dependent sleeps or live endpoints. Test-only hooks may be feature-gated in `bindings/python`/test harness; they must not appear in release builds or bypass policy. If a restricted loopback fixture is used, require explicit authorization and count actual requests; prefer fully in-process fixtures. + +Measure and assert: handler started, handler dropped/abort observed (or explicitly bounded unavoidable completion), no orphan provider requests, shared permit released, client active count drained. Use explicit bounded `asyncio.wait_for` timeouts as a hang guard, never as proof the transport was canceled. Include Python `Future.cancel()` before start as the existing fast path and **after** positive started barrier as the new critical path. Do not confuse Python Task cancellation with Rust future cancellation: trace both ends. + +### WP2 — Concurrent close, cancellation and resource correctness + +Run `aclose()` concurrently with at least one confirmed in-flight operation and a second pending/queued (or rejected by the documented concurrency cap) call. Verify the intended cancellation/error taxonomy (`asyncio.CancelledError` for user cancellation; structured `client_closed` for close-triggered operation), idempotent repeated close, no new calls after close, permit availability for a distinct client, and progress on a subsequent asyncio loop. Test exceptions and interrupted conversions without leaking `ActiveCall`/permit or holding the GIL during async work. Inspect `pyo3-async-runtimes` versioned cancellation semantics and `future_into_py` behavior; if default cancellation does not abort Rust tasks promptly, add an explicit bounded cancellation mechanism with minimal bridge-local changes and a fail-closed no-orphan test. + +Document which in-flight network operations can finish after a cancellation boundary, what deadlines contain them, and whether Python cancellation cancels queued work, running Rust work, or only awaiters. Do **not** claim instantaneous socket termination without a counted transport fixture. A burst stress test should exercise saturation of the global 32-call limit and recovery, independent clients, and cancellation/close races, without flakiness. + +### WP3 — Explicit opt-in default configuration resolution + +**Preserve** `AsyncEggsearch()` existing no-config defaults as documented. Add a clearly named, typed, publicly documented opt-in, recommended `AsyncEggsearch.from_default_config()`, which uses the Rust-owned `eggsearch::core::config::default_config_path()` and `AppConfig::load` policy (or a comparably direct helper) instead of reimplementing platform selection in Python. Preserve `AsyncEggsearch(path)` behavior. Include API/type stub support; expose resolved path only if safe, without leaking credentials or raw config. + +Tests must demonstrate all three cases: no-arg defaults unaffected by a present local config; opt-in factory loading a temporary platform config and enabling an observable safe setting; explicit path override taking precedence. Verify missing standard config matches CLI fallback-to-defaults, invalid TOML / unreadable config reports a deterministic error and never silently weakens security, no network on Python import or configuration construction, and no token contents in exceptions/logs. Use platform-aware fixture isolation and avoid tests modifying users' real config directories. + +If implementation evidence shows preserving no-arg semantics is unsafe or violates a previously documented contract, STOP and record a compatibility decision rather than silently reversing constructor behavior. + +### WP4 — Review, documentation and tests + +Update `bindings/python/tests/test_native.py` (or suitably split behavioural suites), `bindings/python/python/eggsearch/__init__.py`, `_native.pyi`, `bindings/python/python/eggsearch/types.py` only as required, `docs/python-library.md`, `bindings/python/PYTHON_README.md`, `architecture/python-library.md` and test inventories/skills according to repo conventions. Keep Python public errors and Rust `SdkError` codes consistent, preserve type-checking coverage and the ten-operation parity/mode-off fixtures. Do not change M004 wheel inventory or version unless justified by a separate artifact-level compatibility finding. + +## Verification commands and acceptance evidence + +Inspect exact package/test commands before execution, then run and record: + +- `cargo test --locked --all-features --lib sdk` +- `cargo test --locked --all-features --test static_guards` +- `cargo test --locked --all-features --test evidence_contract` +- `make check` +- `rustup run 1.89.0 cargo check --locked --manifest-path bindings/python/Cargo.toml` +- `maturin develop --manifest-path bindings/python/Cargo.toml --locked` inside an isolated venv +- `python -m pytest -q bindings/python/tests` including started-barrier cancel, concurrent close, concurrency-cap recovery and default-config fixtures +- `python -m compileall -q bindings/python/python/eggsearch bindings/python/examples` +- `pyright bindings/python/python/eggsearch bindings/python/examples` + +Do not claim a pass for commands not executed; rerun and record intermittent failures. Root CI, root dependency-policy guards, native Python qualification and packaging will be rerun on the final candidate in M006. + +PASS requires deterministic proof that cancellation after actual Rust work begins has the documented bounded stop behavior; `aclose()` under active work returns without orphan tasks/permits; 32-call saturation recovers; optional default-config loading works using Rust path semantics, preserving no-arg and explicit-path compatibility; existing ten-tool and Mode-Off security parity remains green. + +STOP if the test shows unbounded detached Rust work, swallowed partial failures, unsafe default-config loading, broken CLI/MCP semantics, or a need to bypass policy. Record the failed path and write a narrower follow-up if needed. An immediate `provider_status` cancellation alone cannot close this plan. + +## Required closure record and handoff + +Write `plans/closure/python-native-library/005-status.md` with exact candidate SHA; original-gap-to-regression-case matrix; proof of barrier start and Rust stop/drain/permit release; concurrent close and multi-loop results; config-selection test outcomes; full command results; security/ABI/backward compatibility evidence; limitations and severity-rated residuals. Update roadmap and registry status **in the same closure commit**. M006 becomes ready only after this evidence closes M005. Do not mark M004 closed as a side effect of M005; qualification of **new** bytes is required. diff --git a/plans/implementation/python-native-library/006-m004-exact-sha-distribution-closure.md b/plans/implementation/python-native-library/006-m004-exact-sha-distribution-closure.md new file mode 100644 index 00000000..26ac262c --- /dev/null +++ b/plans/implementation/python-native-library/006-m004-exact-sha-distribution-closure.md @@ -0,0 +1,67 @@ +# M006 — M004 Exact-SHA Distribution Qualification and Closure Reconciliation + +Status: closed (closure `plans/closure/python-native-library/006-status.md` at candidate `faa1da2`, run `38617598636`; M004 and M007 close in the same commit) +Primary class: infrastructure (release qualification and evidence control) +Source: `plans/subsystems/python-native-library-roadmap.md` M006; closes remaining exit conditions of active M004 (`plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md`). +Planning baseline: `main` `1434c1e55ddd92947042cff9918ffc0fe130ed01`, 2026-10-14. +Dependencies: M001–M003 closed (`plans/closure/python-native-library/001-status.md` through `003-status.md`); M005 must close first. Operational: hosted exact-SHA CI, wheel artifact and native consumer evidence; publisher authorization is deliberately outside this plan. +ADR: `plans/adrs/ADR-0005-python-native-library-boundary.md`; normative `plans/003-planning-process.md` sections 2.4, 2.5, 7–10; `plans/000-long-term-specification.md` sections 6, 7, 9. + +## Scope and observed evidence + +M004 already added independently versioned `bindings/python` packaging, `abi3-py311` PyO3 wheels, a 5-native-platform x 4-CPython-version consumer matrix, a clean extracted-sdist rebuild, wheel hashing, and dependency auditing. On baseline `1434c1e55ddd92947042cff9918ffc0fe130ed01`, GitHub Actions `Python native qualification` run `38522548258` (2026-10-14) shows 27 successful jobs: five wheel builds, 20 interpreter/platform smoke jobs, one extracted-sdist rebuild, and one dependency audit. Root `CI` `38522548209`, `Release drift guard` `38522548221`, and `Egress feature qualification` `38522548365` also passed on this SHA. + +**Do not call M004 closed based on these baseline runs.** `plans/closure/python-native-library/004-status.md` is absent; the candidate will change when M005 adds lifecycle tests/config API. GitHub Actions artifacts expire, and green workflow conclusion without package hashes, native consumer install evidence, ABI notes, package naming and closure status is not persistent qualification. The M004 contract deliberately excluded publishing to PyPI. + +This plan supplies a bounded **evidence + final qualification** pass rather than recreating the wheel pipeline or expanding supported architectures. + +## Non-goals and invariants + +No automated publication, PyPI secret/provisioning change, first PyPI release, TestPyPI publishing, changes to existing immutable GitHub release, Eggpack producer regeneration, binary asset matrix change, second Rust SKU, Python provider rewrite, TypeScript/C ABI, PyPy/free-threaded CPython/ARMv7/Windows ARM64 qualification without evidence. Keep root Rust 1.89, root `#![deny(unsafe_code)]`, the ten MCP/CLI contracts, Mode-Off no-remote-dispatch and transport/sanitization policies unchanged. + +M004 closure means artifacts are qualified for a **separately authorized release**, not that `pip install eggsearch` resolves to these unpublished bits on PyPI. + +## Ordered work packages + +### WP1 — Establish post-corrective immutable candidate + +Start only after `plans/closure/python-native-library/005-status.md` is accepted. Determine exact root/bindings Cargo versions, `bindings/python/pyproject.toml` metadata and tag policy, pinned PyO3/maturin versions, `abi3-py311` wheel tags, Cargo lock graph and Git SHA. Confirm supported target list with no extrapolation to unsupported CPython free-threaded builds. Ensure any new commit after qualification **invalidates** existing proof; schedule a new exact-SHA qualifier and do not reuse run `38522548258` as evidence for changed code. + +### WP2 — Require complete native qualification and traceable artifacts + +Execute the configured GitHub `Python native qualification` workflow against the exact candidate with required successful jobs: five native build targets (Linux x86_64/aarch64 with verified manylinux2014 policy, macOS x86_64/arm64, Windows x86_64), and Python 3.11, 3.12, 3.13, 3.14 installed-wheel smoke **on each of the five targets**, independent of Rust being present in consumer virtual environments. Tests must exercise import, canonical inventory, all ten operations in no-network fixtures, concurrency/cancellation/close and trust/Mode-Off behavior. Record wheel/platform tag, ABI, package metadata, source SHA, hosted run/job IDs, filename, SHA-256, and GitHub upload artifact digest. Do not assert that cross-compilation alone constitutes native qualification. + +Check that Python 3.14 stable-ABI wheels work and that current supported Python matrix is accurate; no PyPy/free-threaded claim. Linux aarch64 runner execution is not proof for ARMv7. Any missing target stays explicitly unsupported or **blocks** M004 if advertised. + +### WP3 — Rebuild independent sdist and audit supply chain + +Build `maturin build --release --sdist --locked`, extract into a clean directory outside checkout, ensure it carries the binding crate + root Rust path dependency + lockfiles/license/readme, and rebuild **offline with prefetched locked Cargo crates**, then clean-install and run Python consumer suite. Validate no private paths/secrets/credentials/tracked build products in distributions. Run Cargo dependency policy, Python package dependency audit, action SHA pin checks and project metadata consistency. Verify `eggsearch` distribution name availability/ownership immediately before publication rather than making a permanent claim from an October 14 PyPI 404. + +Capture durable SHA-256 digests (not only ephemeral 14-day artifact links), package wheel tags and `abi3` decisions. If clean sdist extraction requires source checkout or network fetch despite prefetched Cargo crates, record as a blocker. + +### WP4 — Root compatibility and release boundary + +Re-run `make check`, `make release-check` where environment permits, `cargo publish --dry-run --locked` and `cargo package --list` for root to prove that the independent Python binding has not changed the root published Cargo package or Eggpack release inputs. Confirm there is no Python wheel in the canonical seven-target binary assets, and that existing release workflow remains generated/owned by Eggpack. Run no-network Mode-Off SDK/Python agent fixture and source-trust tests on the final candidate. Fail closed on unknown status/partial failures. No publication or immutable release edits. + +### WP5 — Close M004 and reconcile planning truth + +Create `plans/closure/python-native-library/004-status.md` on the final candidate containing a requirement-to-evidence matrix, exact SHA, completed platform/interpreter/job table, five wheel filenames/tags/hashes and sdist hash, extracted-sdist clean rebuild and import proof, Python consumer fixture run, provenance/action/dependency evidence, binary/root compatibility gates, publication status (unpublished), unsupported targets, observed limitations, and severity-classified residual findings. Write `plans/closure/python-native-library/006-status.md` as a small final reconciliation cross-referencing the real M004, M005, and M007 records, not duplicating their test tables. The final candidate is the M007 repair SHA, not any pre-fix bytes; pre-fix runs stay on record as superseded evidence showing the deficiency, never as qualification. + +Update `plans/subsystems/python-native-library-roadmap.md`, `plans/registry.md`, `docs/python-library.md`, package README, release guidance and any test inventory/architecture docs that actually changed. Close M004 and M006 **only together with the required exact-candidate evidence and both closure records in the same commit**; do not claim the workstream is published or prematurely archive still-actionable documents. If a high/medium gap appears, leave M004 active/blocked, record the deficiency, and issue another corrective rather than writing fictional green closure. + +## Verification commands and exit + +Canonical gates (record versions and actual invocation): +- `make check` and `make release-check` (report environment constraints instead of inventing a result) +- `cargo package --list`; `cargo publish --dry-run --locked` +- `rustup run 1.89.0 cargo check --locked --manifest-path bindings/python/Cargo.toml` +- `maturin build --manifest-path bindings/python/Cargo.toml --release --sdist --locked --out dist` +- `maturin build --manifest-path bindings/python/Cargo.toml --release --locked --out dist` for all five target builders +- `python -m pytest -q bindings/python/tests`; `pyright bindings/python/python/eggsearch bindings/python/examples`; `python -m compileall -q bindings/python/python/eggsearch bindings/python/examples` +- Hosted `.github/workflows/python-native-qualification.yml` on the **same exact candidate**, including wheel builds, 20 wheel-smoke jobs, sdist rebuild and dependency audit; main CI/release drift/egress qualification. + +PASS means all M004 operational requirements have SHA-specific logs/attestations/digests, all claimed wheels install natively and pass Python tests in clean consumer venvs, sdist rebuilds outside checkout, root release compatibility remains proven, and closure records/registry are synchronized. Fail if any advertised wheel lacks native smoke, digests cannot be recovered, a candidate SHA differs, Rust/Python runtime regression exists, or documentation confuses qualification with PyPI availability. + +## Handoff and closure + +M006 is blocked until M005 closes. When M005 closes, set M006 ready and execute against the **post-M005** candidate. The final closure must explicitly retain M001–M003 historical evidence, mark M005 closed, mark M004 and M006 closed only after qualification, and state `Published to PyPI: no` unless independently authorized and verifiably accomplished. Separate package publishing can be planned later without reopening successful qualification. diff --git a/plans/implementation/python-native-library/007-windows-safe-read-corrective.md b/plans/implementation/python-native-library/007-windows-safe-read-corrective.md new file mode 100644 index 00000000..9dde7045 --- /dev/null +++ b/plans/implementation/python-native-library/007-windows-safe-read-corrective.md @@ -0,0 +1,98 @@ +# M007 — Windows Safe-Read Corrective for Local Content Access + +Status: closed (closure `plans/closure/python-native-library/007-status.md` at candidate `faa1da2`, proven by run `38617598636` Windows smokes) +Primary class: invariant (local path-jail reads must behave identically on all supported platforms; fail-closed semantics preserved) +Source: `plans/subsystems/python-native-library-roadmap.md` M007; corrective to M004/M006 qualification, not a rewrite of closed M001–M003/M005 records. +Planning baseline: `plan/python-native-closure-correctives` `d5e4dcb`, 2026-10-14. +Dependencies: M001–M003 and M005 closed; M004 active; M006 active and awaiting this candidate for its final matrix. No new operational dependencies beyond the existing hosted qualification workflow. +ADR: `plans/adrs/ADR-0005-python-native-library-boundary.md`; normative `plans/003-planning-process.md` sections 2.4, 2.5, 7–10. + +## Scope and observed evidence + +M006 qualification of `c8bd030` (run `38608502185`) failed 3 config tests +on all Windows smoke jobs. Test-isolation repair (`748ae59`, per-test CWD +plus fallback filename tolerance) fixed 2 of 3; run `38611932990` left +exactly `test_explicit_path_overrides_default_config` failing on all four +Windows interpreters while 16/16 Linux/macOS smoke jobs passed. +Instrumented run `38614592873` (temporary diagnostics, since reverted) +showed the Windows client loading the correct config (`mojeek=true`, +`repo=default-workspace`, card present with `local_trusted` provenance) +but with no content snippet, so `"alpha" in snippet` failed. + +Cause, confirmed by code inspection: `safe_open_relative()` in +`src/meta/safe_open.rs`, `#[cfg(not(unix))]` branch, begins with +`File::open(root)` where `root` is always a workspace directory. Opening a +directory with `File::open` fails on Windows (CreateFile without +`FILE_FLAG_BACKUP_SEMANTICS` returns access-denied), so **every** +`safe_read_file`/`safe_open_relative` call fails on Windows. Local search +enumeration still finds files (cards exist, trust intact), but all content +reads fail and every Windows local snippet is `None`. The same branch also +follows symlinked roots, where the unix branch rejects them (`O_NOFOLLOW` +semantics) — the repair aligns both. No Rust test executes on Windows in CI +(the Windows job is `cargo check` only), so the defect was invisible until +the Python smoke matrix asserted snippet content. + +Blast radius on Windows: local search scoring degrades to path/role-only +and snippets are absent; `security_search.rs:1538` uses the same helper. +No crash, no jail escape, no cross-platform behavior change: the repair +keeps fail-closed semantics and the existing error taxonomy (no new error +variants, no API change). + +## Non-goals and invariants + +No change to the jail policy (relative-only, no traversal, no hidden files +unless configured, size caps, `follow_symlinks=false` default), no new +`SafeOpenError` variants, no MCP/CLI contract change, no provider, +transport, sanitization-bypass, or Eggpack change. Root Rust 1.89 and +`#![deny(unsafe_code)]` unchanged. M004 stays active until this fix is +re-qualified; no publication. + +## Ordered work packages + +### WP1 — Repair the non-unix root check + +In `src/meta/safe_open.rs`, replace the non-unix `File::open(root)` +existence probe with a `std::fs::symlink_metadata(root)` check that +requires a directory and rejects symlinked roots when +`follow_symlinks=false` (unix parity). Keep the remainder of the +component walk byte-identical. Preserve every error variant currently +used on that path. + +### WP2 — Regression tests + +Add unit tests in `src/meta/safe_open.rs`: round-trip read of a file +through a directory root, and rejection of a file-as-root. Both pass on +Linux before and after (shared logic guard); on Windows they fail before +and pass after (stated, not executed here — Linux CI has no Windows test +execution). The hosted Windows smoke matrix (`test_explicit_path_overrides_default_config` +snippet asserts) is the Windows proof. + +### WP3 — Re-qualify and close out + +Run local gates (`cargo fmt --check`, clippy, affected Rust suites, Python +28-test suite), push, and record the full hosted `Python native +qualification` matrix on the exact repair candidate. M006 then finalizes +M004/M006 closure on those bytes; this plan closes with its own +`007-status.md` in the same commit. + +## Verification commands and exit + +- `cargo fmt --check` +- `cargo clippy --locked --all-targets --all-features -- -D warnings` +- `cargo test --locked --all-features safe_open local_backend sdk` +- `maturin develop --manifest-path bindings/python/Cargo.toml --locked` + `python -m pytest -q bindings/python/tests` +- Hosted `.github/workflows/python-native-qualification.yml` on the exact candidate: 5/5 builds, 20/20 smokes (in particular the 4 Windows jobs with snippet asserts), sdist rebuild green; audit job outcome recorded with the standing rustls residual. + +PASS means the repair candidate shows Windows snippet asserts green with no +other matrix movement, local gates pass, and the deficiency record plus +this plan close together with M004/M006. Fail closed on any new red. + +## Handoff and closure + +M007 closes with `plans/closure/python-native-library/007-status.md` +recording the defect, the repair diff, test evidence on both platforms +(including the stated Windows-only observability limit), the exact repair +SHA, and the re-qualification run ID. M004 and M006 close on the same +repair candidate in the same commit; the M006 plan already anticipates this +routing ("issue another corrective rather than writing fictional green +closure"). diff --git a/plans/implementation/python-native-library/008-rustls-and-symlink-containment-requalification.md b/plans/implementation/python-native-library/008-rustls-and-symlink-containment-requalification.md new file mode 100644 index 00000000..cc706f9d --- /dev/null +++ b/plans/implementation/python-native-library/008-rustls-and-symlink-containment-requalification.md @@ -0,0 +1,141 @@ +# Python Native Library M008 — Advisory Remediation and Symlink-Containment Re-qualification + +Status: closed (closure `plans/closure/python-native-library/008-status.md` at candidate `067468a26c21716dd1063d67679768c6d794a2b9`, qualifying run `38641009246` 27/27 including audit) +Roadmap: `plans/subsystems/python-native-library-roadmap.md` M008 +Class: invariant +Related: ADR-0005 (`plans/adrs/ADR-0005-python-native-library-boundary.md`); +closures `plans/closure/python-native-library/004-status.md`, +`006-status.md`, `007-status.md` (retained as history, never amended). +Baseline: `8f632b3` (M004/M006/M007 closed; `dependency-policy` red on +upstream rustls advisories; Windows symlink-following gap contained only +in prose). + +## Objective + +Resolve the two residuals named in the M004/M006/M007 closures, then +re-qualify the Python distribution on the exact repair candidate so the +closure records reconcile to fully green evidence: + +1. Remediate upstream rustls advisories RUSTSEC-2026-0346/0347/0348/0349 + against locked rustls 0.23.45 (lockfile bump to the fixed 0.23 line, + no manifest change, no waiver). +2. Close the deterministic Windows filesystem hole M007 left open: the + non-unix walk in `src/meta/safe_open.rs` follows symlinks + (`std::fs::metadata` at every component) while the unix branch rejects + them, so `follow_symlinks=false` is not enforced on Windows. Reject + symlinks present at walk time with the existing `SymlinkDetected` + taxonomy, and explicitly contain the residual check-to-open race + (Windows std offers no atomic no-follow open) in the closure record. +3. Rerun the hosted Python native qualification on the exact repair + candidate and reconcile M004/M006 documentation to the resulting + green evidence. + +## Non-goals + +- Publication to PyPI (still separately authorized); no tag, no release + mutation, no Eggpack producer change. +- Chromiumoxide reqwest removal (repository-hardening M006 owns that + blocker); no new Windows-only dependencies (no `windows-sys` atomic + open — the residual race is contained, not engineered away). +- New `SafeOpenError` variants, taxonomy changes, unix-branch behavior + changes, or root-symlink policy changes (unix follows root symlinks; + Windows keeps parity). +- Free-threaded Python, ARMv7, Windows arm64, or any target-matrix change. +- Amending `001-status.md`–`007-status.md`; reconciliation lives in the + new `008-status.md` with a supersession table. + +## Current implementation evidence + +- `cargo deny check advisories` on the baseline fails exactly on + RUSTSEC-2026-0346–0349, each with solution `Upgrade to >=0.23.46`. + Reverse dependents of rustls 0.23.45: `eggfetch-core 0.2.2`, + `eggress-* 1.0.12`, `hyper-rustls 0.27.9`, `tokio-rustls 0.26.4`, + plus the direct `rustls = "0.23"` requirement in `Cargo.toml`. +- `src/meta/safe_open.rs` `#[cfg(not(unix))]` branch uses + `std::fs::metadata` (follows links) for root, intermediate, and final + components; `File::open` follows a swapped-in link. All local content + reads (`src/meta/local_backend.rs`, `src/meta/security_search.rs:1538`) + flow through this path. +- CI `windows-portability` runs `cargo check --locked --all-features + --target x86_64-pc-windows-msvc` only: no Rust test executes on + Windows. The Python smoke matrix (5 targets x CPython 3.11-3.14) is the + only Windows execution gate. + +## Invariants that cannot regress + +- Ten MCP tools, schemas, CLI behavior, deterministic FNV-1a IDs, + sanitization tiers, bounded-I/O helpers, `follow_symlinks=false` + default, `SafeSymlinkFollowingUnsupported` for `follow_symlinks=true` + on non-Linux, root `#![deny(unsafe_code)]` with the three-file allow + inventory unchanged, Rust 1.89 MSRV, lockfile-only dependency change + (manifests, versions, registries untouched). +- Static guards: `no_path_based_reads_in_safe_open`, + `crate_roots_deny_unsafe_code`, + `unsafe_allow_inventory_is_explicit_and_minimal`, + `planning_status_consistency_no_closed_range_shorthand`. + +## Expected production changes + +- `Cargo.lock`: rustls 0.23.45 -> 0.23.46 (checksum update only). +- `docs/dependency-security.md`: remediation line follows the lockfile. +- `src/meta/safe_open.rs` non-unix branch only: `symlink_metadata` at + each component, `SymlinkDetected` on links (message parity with the + unix branch), plus `cfg(any(unix, windows))` regression tests for + final and intermediate link rejection. +- `bindings/python/tests/test_native.py`: one Windows-only behavioral + smoke test (symlinked file content never surfaces; real file snippet + intact; skip with reason when symlink privilege is unavailable). +- `architecture/local-workspace.md`, `architecture/hardening.md`: scope + the race-free/TOCTOU claims to the atomic primitives and state the + Windows containment explicitly. + +## Ordered work packages + +- WP1 rustls remediation: bump, `cargo deny check`, policy script, + MSRV build proof, deny/policy evidence recorded. +- WP2 symlink containment: fix, Rust tests + (`safe_open` lib suite green on Linux), Windows-target check compile, + Python smoke test green locally on macOS/Linux (skipped) plus review. +- WP3 local gates: `make check` fully green including + `dependency-policy`; `make packaging-check`; Python develop suite + (28 passed) and `compileall`; `cargo package --list` / + `cargo publish --dry-run --locked` unchanged shape. +- WP4 hosted exact-commit qualification: push repair candidate, + dispatch the Python native qualification workflow, require 5/5 wheel + builds, 20/20 installed-wheel smokes (4/4 Windows incl. the new + symlink test), extracted-sdist offline rebuild with clean-venv + consumer run, and a green audit job; harvest digests and job IDs. +- WP5 closure reconciliation: `008-status.md` with evidence matrix and + the supersession table (new candidate/run supersede `faa1da2` / + `38617598636` as the qualification basis; old records stand as + history); roadmap + registry + architecture handoff to M008 closed; + commit and push on the workstream branch. + +## Verification commands + +Focused: `cargo deny check`, `cargo test --locked --all-features +--lib safe_open`, `cargo check --locked --all-features --target +x86_64-pc-windows-msvc` (when the target is installed), +`python -m pytest -q bindings/python/tests` (post-`maturin develop`). +Broad: `make check`, `make packaging-check`, +`./packaging/check-contract.sh`. + +## Acceptance and stop conditions + +M008 closes when `make check` passes end to end (including a green +`dependency-policy`), the hosted qualification on the exact repair +candidate is green in all jobs including audit, and `008-status.md` +plus roadmap/registry/architecture updates land in one closure commit. +Stop and escalate rather than waive: if the rustls bump breaks the +MSRV build, route a waiver through the repository dependency process +instead of adding a `deny.toml` ignore here; if the Windows smoke +symlink test cannot execute (privilege), record the skip counts and let +the Rust tests plus review stand as the containment evidence. + +## Required closure evidence + +Candidate SHA, qualifying run id with per-job outcome table, wheel/sdist +digests, `cargo deny check` output, `make check` / `make +packaging-check` results, consumer-run counts per target, residual list +(check-to-open race containment, any skipped Windows symlink execution), +and the M004/M006/M007 supersession table. diff --git a/plans/registry.md b/plans/registry.md index f504db6b..2d3f44c3 100644 --- a/plans/registry.md +++ b/plans/registry.md @@ -42,7 +42,7 @@ Historical `phase-*.md` plans and pre-migration roadmaps are archived under | MCP tool-surface consolidation | closed | `plans/subsystems/tool-surface-consolidation-roadmap.md` | M001-M007 closed | M001-M007 closed. Current control points `plans/closure/mcp-tool-surface-consolidation/001-status.md` through `007-status.md`. | | Dependency evidence hardening | closed | `plans/subsystems/dependency-evidence-hardening-roadmap.md` | M010 corrective closed | M001-M010 closed. Current control point `plans/closure/dependency-evidence-hardening/010-status.md`; implementation `23676cc`, CI `36253474005`. | | Repository security, supply-chain, and maintenance hardening | active | `plans/subsystems/repository-hardening-roadmap.md` | M001/M002/M007/M008/M009/M010 closed; M006 blocked | Scheduled dependency-security run `38290328560` succeeded; M010 records M001/M007 closure. M002 closed by M009; M006 remains externally blocked on chromiumoxide. | -| Python native library for agents | active (M001-M003 closed; M004 active) | `plans/subsystems/python-native-library-roadmap.md` | M003 closed at `plans/closure/python-native-library/003-status.md`; M004 qualification work is active | ADR-0005 accepted; Python-only in-process SDK workstream. Publication remains separately authorized. | +| Python native library for agents | closed (M001-M008 closed; unpublished by design) | `plans/subsystems/python-native-library-roadmap.md` | M008 closed at `067468a`, qualifying run `38641009246` (27/27 incl. audit) | ADR-0005 accepted. Runs `38522548258`, `38608502185`/`38611932990`/`38614592873`, `38637266178`, and `38638066765` are historical/deficiency/supporting evidence only. Publication separately authorized. | ## Dependency-ready implementation plans @@ -87,7 +87,11 @@ Historical `phase-*.md` plans and pre-migration roadmaps are archived under | Python native library | M001 shared Rust SDK facade/parity | **closed** | `plans/implementation/python-native-library/001-shared-sdk-facade-and-parity.md` | Candidate `5f686d02e15daa2c5db7ad6d889c06eaff3ceeee`; closure `plans/closure/python-native-library/001-status.md`. | | Python native library | M002 PyO3 asyncio binding/lifecycle | **closed** | `plans/implementation/python-native-library/002-pyo3-async-execution-and-lifecycle.md` | Candidate `2392e72`; closure `plans/closure/python-native-library/002-status.md`; bounded conversion, cancellation/close, and ten-operation consumer smoke. | | Python native library | M003 Python agent API, types, parity | **closed** | `plans/implementation/python-native-library/003-agent-api-types-and-compatibility.md` | Candidate `ca4d0d6`; closure `plans/closure/python-native-library/003-status.md`; typed ten-method API, schema coverage, Mode-Off agent flow. | -| Python native library | M004 wheels, sdist and release qualification | **active** | `plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md` | M003 closed; qualify clean wheel/sdist installs and exact native artifacts. No publication. | +| Python native library | M004 wheels, sdist and release qualification | **closed** | `plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md` | Candidate `faa1da2`; closure `plans/closure/python-native-library/004-status.md`; 5/5 builds, 20/20 smokes, offline sdist rebuild, digests; no publication. | +| Python native library | M005 in-flight cancellation/config corrective | **closed** | `plans/implementation/python-native-library/005-inflight-cancellation-and-default-config-corrective.md` | Candidate `baf00cf`; closure `plans/closure/python-native-library/005-status.md`; confirmed-in-flight cancellation/close with permit recovery plus opt-in default-config path. | +| Python native library | M006 M004 exact-SHA closure and registry reconciliation | **closed** | `plans/implementation/python-native-library/006-m004-exact-sha-distribution-closure.md` | Candidate `faa1da2`; closure `plans/closure/python-native-library/006-status.md`; qualifying run `38617598636`. | +| Python native library | M007 Windows safe-read corrective | **closed** | `plans/implementation/python-native-library/007-windows-safe-read-corrective.md` | Candidate `faa1da2`; closure `plans/closure/python-native-library/007-status.md`; non-unix root check repaired, proven by 4/4 Windows smokes. | +| Python native library | M008 advisory remediation and symlink-containment re-qualification | **closed** | `plans/implementation/python-native-library/008-rustls-and-symlink-containment-requalification.md` | Candidate `067468a`; closure `plans/closure/python-native-library/008-status.md`; rustls 0.23.46 in both lockfiles, walk-time link rejection with fail-closed unix pin, 27/27 re-qualification with green audit. | Sequencing overview: `plans/implementation/mcp-tool-surface-consolidation/000-overview-and-sequencing.md`. Handoff checklist: `plans/implementation/mcp-tool-surface-consolidation/008-implementation-handoff-checklist.md`. @@ -100,7 +104,7 @@ Corrective closure sequence: `plans/implementation/dependency-evidence-hardening ## Current execution order and dependency gates -**Python native library gate:** ADR-0005 authorizes a narrow direct Rust SDK facade and Python PyO3 native package without broad Rust module stability. M001-M003 are closed with exact-candidate evidence; M004 distribution qualification is active. Python package publication requires separate authorization and does not alter Eggpack binary assets. +**Python native library gate:** ADR-0005 authorizes a narrow direct Rust SDK facade and Python PyO3 native package without broad Rust module stability. M001-M008 are closed at `067468a` with fully-green qualifying run `38641009246` (5/5 builds, 20/20 smokes, offline sdist rebuild, green audit, digests in `008-status.md`). Earlier runs are historical/deficiency/supporting evidence only. Publishing still needs separate authorization; Eggpack binary assets remain unchanged. **Tool-surface gate:** M001-M007 closed. Do not run evaluation against pre-consolidation bytes and present it as consolidation @@ -172,7 +176,7 @@ release that makes HTTP discovery / reqwest optional without regressing | Repository hardening | active; M008 closed, M007 conditional, M002 conditional, M006 blocked | M008 closed at `44f38ab` (`008-status.md`); M007 remains conditional evidence control point. Historical closure records `001-status.md` through `008-status.md`. | | CodeGG legacy search parity | closed; M001 through M005 closed | `plans/closure/codegg-legacy-search-parity/005-status.md`; prior control points `plans/closure/codegg-legacy-search-parity/004-status.md` and `003-status.md`; implementations `e9103b4` (M001), `5233315` (M002), `8e5ec75` (M003), `81a2e49` (M004 release-prep), `32769d6` (M005 fix-prep); historical provider-parity closure retained, new Mode-Off locality issue separately registered | | Mode-Off local repository dispatch corrective | active; M001 closed; M002 and M003 blocked | M001 closure `plans/closure/mode-off-repository-locality-corrective/001-status.md` at `cc0c9c7`; M002 blocked pending M003 and immutable release evidence; M003 implementation `fcb1fb5`, blocked pending default-branch hosted qualification, closure `plans/closure/mode-off-repository-locality-corrective/003-status.md`. CodeGG paired requalification remains consumer-owned. | -| Python native library | active; M001-M003 closed, M004 active | `plans/subsystems/python-native-library-roadmap.md`; M003 control point `plans/closure/python-native-library/003-status.md` | +| Python native library | closed; M001-M008 closed | `plans/subsystems/python-native-library-roadmap.md`; closure records `001-status.md`–`008-status.md`; M008 closed at `067468a`, run `38641009246`; unpublished by design | ## Closure rule diff --git a/plans/subsystems/python-native-library-roadmap.md b/plans/subsystems/python-native-library-roadmap.md index 46fd8bcd..734fcc7c 100644 --- a/plans/subsystems/python-native-library-roadmap.md +++ b/plans/subsystems/python-native-library-roadmap.md @@ -1,6 +1,6 @@ # Python Native Library Roadmap -Status: active (M001-M003 closed; M004 active for distribution qualification) +Status: closed (M001-M008 closed; unpublished by design, publication separately authorized) Planning baseline: `eggstack/eggsearch` `main` at `f386b07d7adaa06bbb4a51a9d7e2eecf971c238b` (2026-10-13, Cargo package version 0.4.3) Primary outcome: in-process async Python library consumable by coding and research agents. @@ -10,7 +10,7 @@ References: `plans/000-long-term-specification.md` sections 1-4, 6-9; `plans/001 Provide a native Python library wrapping Eggsearch's Rust search/fetch/evidence engine. Python users must be able to `pip install` a compatible wheel, instantiate an asyncio client, execute any of the ten established tools without spawning the MCP server, share search/cache/provider state across concurrent calls, and receive structured, trustworthy results and repairable errors. -Ownership: `src/sdk/` owns the narrow direct-operation/application API and shared client lifecycle, while `src/mcp/tools/` remains the single semantic tool execution owner. `bindings/python/` owns PyO3 integration and Python-only ergonomics; it MUST NOT implement its own provider selection, policy, SSRF, fetch, sanitization, result projection, stable-ID algorithm, or ranking. Root default binary/deployment packaging remains owned by existing release and Eggpack paths. +Ownership: `src/sdk.rs` owns the narrow direct-operation/application API and shared client lifecycle, while `src/mcp/tools/` remains the single semantic tool execution owner. `bindings/python/` owns PyO3 integration and Python-only ergonomics; it MUST NOT implement its own provider selection, policy, SSRF, fetch, sanitization, result projection, stable-ID algorithm, or ranking. Root default binary/deployment packaging remains owned by existing release and Eggpack paths. ## Invariants (must hold in every milestone) @@ -39,20 +39,32 @@ At the planning baseline, `src/lib.rs` promised only MCP/CLI stability, `src/mcp `M003 (capability: ergonomic Python agent API, types, examples, semantic parity)` -> closed at `plans/closure/python-native-library/003-status.md` `M004 (infrastructure + operational: wheel/sdist/release qualification)` - -> active - -M001-M003 are closed and M004 is dependency-ready. Platform qualification remains required; no cross-repository or Mode-Off release blocker is waived. Re-run the exact Mode-Off regressions, with separate operational gating for publication if existing immutable-release qualification remains blocked. + -> closed at `plans/closure/python-native-library/004-status.md` +`M005 (invariant: in-flight cancellation/close and CLI-config opt-in corrective)` + -> closed at `plans/closure/python-native-library/005-status.md` +`M006 (infrastructure: M004 exact-SHA artifact and closure reconciliation)` + -> closed at `plans/closure/python-native-library/006-status.md` +`M007 (invariant: Windows safe-read corrective for local content access)` + -> closed at `plans/closure/python-native-library/007-status.md` +`M008 (invariant: rustls advisory remediation and Windows symlink-containment re-qualification)` + -> closed at `plans/closure/python-native-library/008-status.md` + +M001-M008 are closed. M004/M006/M007 closed at `faa1da2` (run `38617598636`); M008 then resolved the two residuals those closures named — upstream rustls advisories RUSTSEC-2026-0346–0349 and the non-unix symlink-following gap — and re-qualified the repair candidate `067468a` in fully-green run `38641009246`, reconciled in `008-status.md` without amending the earlier records. No cross-repository or Mode-Off release blocker is waived. Publication remains separately authorized. | Milestone | Class | Status | Plan | User-facing exit | |---|---|---|---|---| | M001 direct execution facade | infrastructure | **closed** | `plans/implementation/python-native-library/001-shared-sdk-facade-and-parity.md` | Closure `plans/closure/python-native-library/001-status.md`; candidate `5f686d02e15daa2c5db7ad6d889c06eaff3ceeee`; ten handler parity cases, Mode-Off zero-dispatch counter, and `make check` passed | | M002 native asyncio bridge | capability | **closed** | `plans/implementation/python-native-library/002-pyo3-async-execution-and-lifecycle.md` | Closure `plans/closure/python-native-library/002-status.md`; candidate `2392e72178aa249445a64bd629caac8219353499`; ten-operation asyncio smoke, bounded conversion, cancellation and close | | M003 Python agent ergonomics and safety | capability | **closed** | `plans/implementation/python-native-library/003-agent-api-types-and-compatibility.md` | Closure `plans/closure/python-native-library/003-status.md`; candidate `ca4d0d6`; typed methods, schema coverage, no-network Mode-Off agent flow | -| M004 Python distribution and qualification | infrastructure | **active** | `plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md` | Clean wheel/sdist installation and reproducible native consumer smoke on qualified targets | +| M004 Python distribution and qualification | infrastructure | **closed** | `plans/implementation/python-native-library/004-wheel-sdist-and-release-qualification.md` | Closure `plans/closure/python-native-library/004-status.md`; candidate `faa1da2`; 5/5 builds, 20/20 smokes, offline sdist rebuild, digests recorded | +| M005 lifecycle and configuration corrective | invariant | **closed** | `plans/implementation/python-native-library/005-inflight-cancellation-and-default-config-corrective.md` | Closure `plans/closure/python-native-library/005-status.md`; candidate `baf00cfc8e72f5975010f62e40d5d56985f77c78`; deterministic confirmed-in-flight cancellation/close + permit recovery; explicit default-config opt-in and backwards-compatible no-arg semantics | +| M006 M004 exact-SHA closure reconciliation | infrastructure | **closed** | `plans/implementation/python-native-library/006-m004-exact-sha-distribution-closure.md` | Closure `plans/closure/python-native-library/006-status.md`; qualifying run `38617598636` on `faa1da2`; unpublished unless separately approved | +| M007 Windows safe-read corrective | invariant | **closed** | `plans/implementation/python-native-library/007-windows-safe-read-corrective.md` | Closure `plans/closure/python-native-library/007-status.md`; repaired `safe_open_relative` non-unix root handling; proven by 4/4 Windows smokes on `faa1da2` | +| M008 advisory remediation and symlink-containment re-qualification | invariant | **closed** | `plans/implementation/python-native-library/008-rustls-and-symlink-containment-requalification.md` | Closure `plans/closure/python-native-library/008-status.md`; candidate `067468a`; rustls 0.23.46 in both lockfiles, walk-time link rejection, 27/27 re-qualification with green audit | ## Cross-cutting design requirements -**Configuration:** default and explicit config-file loading must use `AppConfig` and its validation. JSON/dict overrides are optional and MUST not allow privileged policy bypass. Credentials remain referenced by environment names. Python client owns or borrows an immutable effective configuration; avoid global mutable knobs. +**Configuration:** currently `AsyncEggsearch()` uses `AppConfig::default()` rather than the CLI's `default_config_path()`. M005 must preserve that documented no-arg behavior, add an explicit typed opt-in for Rust-owned default-config-path loading, and test parity/security with the CLI and with an explicit config path. All forms must use `AppConfig` validation, keep credentials environment-indirected and avoid mutable global overrides. **Lifecycle and async:** single shared Tokio runtime/bridge is reused, never constructed once per operation. Ref-counted state may be shared by clients; per-instance close rejects new work, drains/cancels bounded in-flight work and is idempotent. Failure to cancel at the Python layer must not imply unbounded Rust work. @@ -68,6 +80,6 @@ M001-M003 are closed and M004 is dependency-ready. Platform qualification remain ## Verification and closure -Each milestone adds deterministic no-network fixtures and the `make check` root gate; package milestones additionally use Python pytest/type checks and clean installation. Track contract parity for all ten tools plus failure/warning/Mode-Off cases, not only `web_search` happy-path. Run `cargo package --list`/`cargo publish --dry-run --locked` for root after packaging changes. Every milestone creates `plans/closure/python-native-library/NNN-status.md` with source SHA, completed evidence matrix, tests and outcomes, open findings, compatibility/security evidence and final status, and updates this roadmap + registry in the same closure commit. M001-M003 are closed by their corresponding closure records; M004 is active. +Each milestone adds deterministic no-network fixtures and the `make check` root gate; package milestones additionally use Python pytest/type checks and clean installation. M005 adds a barrier-controlled *running Rust future* cancellation/close regression, and M006 requalifies all wheels and the extracted sdist at the final exact SHA, recording long-lived hashes and native consumer evidence. Track contract parity for all ten tools plus failure/warning/Mode-Off cases, not only `web_search` happy-path. Run `cargo package --list`/`cargo publish --dry-run --locked` for root after packaging changes. Every milestone creates `plans/closure/python-native-library/NNN-status.md` with source SHA, completed evidence matrix, tests and outcomes, open findings, compatibility/security evidence and final status, and updates this roadmap + registry in the same closure commit. M001-M008 are closed by their corresponding closure records (`008-status.md` at `067468a` joining `001-status.md`–`007-status.md`); M008 re-qualified the repair candidate and reconciled without amending the earlier records. -Do not declare Python availability merely because a PyO3 extension compiled; user-visible capability requires a real Python consumer smoke. Do not publish wheels until exact-SHA qualification and separate publisher approval. Branch and PR handoff is acceptable; do not merge planning into unrelated repository-hardening or Mode-Off release work. +Do not declare Python availability merely because a PyO3 extension compiled; user-visible capability requires a real Python consumer smoke. Do not treat the green M004 baseline run as proof for later bytes or as a closure record. M004/M006/M007 closed together on the M007 repair candidate `faa1da2` with qualifying run `38617598636`; M008 superseded that basis with the fully green re-qualification of `067468a` (run `38641009246`). Do not publish wheels until exact-SHA qualification and separate publisher approval. Branch and PR handoff is acceptable; do not merge planning into unrelated repository-hardening or Mode-Off release work. diff --git a/skills/eggsearch-dev/SKILL.md b/skills/eggsearch-dev/SKILL.md index 1807b5b5..7e55e1fa 100644 --- a/skills/eggsearch-dev/SKILL.md +++ b/skills/eggsearch-dev/SKILL.md @@ -66,7 +66,7 @@ cargo bench --locked --all-features --bench perf --no-run # characterization on | `tests/browser_profiles.rs` | `browser` | Browser profile management | | `tests/browser_transport.rs` | `browser` | Browser transport orchestration | | `tests/mcp_http.rs` | `all-features` | Loopback Streamable HTTP lifecycle, bounds, identity, and shutdown | -| `bindings/python/tests/test_native.py` | CPython 3.11–3.14 | Native asyncio bridge, input bounds, cancellation and close lifecycle | +| `bindings/python/tests/test_native.py` | CPython 3.11–3.14 | Native asyncio bridge, input bounds, started-barrier cancellation with Rust drop/permit recovery, concurrent close and 32-call saturation recovery, and default-config selection | This table is representative, not exhaustive — 78 test suites exist. Full per-suite inventory lives in `docs/test-inventory.md`. diff --git a/src/meta/safe_open.rs b/src/meta/safe_open.rs index bbd4df24..65c4ecd6 100644 --- a/src/meta/safe_open.rs +++ b/src/meta/safe_open.rs @@ -130,7 +130,9 @@ fn fstat_is_regular_and_size(fd: libc::c_int) -> Result<(bool, u64), std::io::Er /// primitive is available. /// /// On non-Unix platforms, `follow_symlinks=true` returns -/// `SafeSymlinkFollowingUnsupported`. +/// `SafeSymlinkFollowingUnsupported`, and components that are symlinks +/// at walk time are rejected with `SymlinkDetected` when following is +/// disabled. #[allow(unsafe_code)] pub fn safe_open_relative( root: &Path, @@ -342,8 +344,14 @@ pub fn safe_open_relative( return Err(SafeOpenError::SafeSymlinkFollowingUnsupported); } - let _ = File::open(root) + let root_meta = std::fs::metadata(root) .map_err(|e| SafeOpenError::RootOpenFailed(format!("{}: {}", root.display(), e)))?; + if !root_meta.is_dir() { + return Err(SafeOpenError::RootOpenFailed(format!( + "root is not a directory: {}", + root.display() + ))); + } let mut accumulated = root.to_path_buf(); @@ -360,13 +368,20 @@ pub fn safe_open_relative( accumulated = accumulated.join(name); if is_last { - let meta = std::fs::metadata(&accumulated).map_err(|e| { + let meta = std::fs::symlink_metadata(&accumulated).map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { SafeOpenError::NotFound(accumulated.display().to_string()) } else { SafeOpenError::Io(e) } })?; + if meta.file_type().is_symlink() { + return Err(SafeOpenError::SymlinkDetected(format!( + "{}/{}", + root.display(), + relative + ))); + } if !meta.is_file() { return Err(SafeOpenError::NotAFile); } @@ -389,13 +404,16 @@ pub fn safe_open_relative( }); } - let entry_meta = std::fs::metadata(&accumulated).map_err(|e| { + let entry_meta = std::fs::symlink_metadata(&accumulated).map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { SafeOpenError::NotFound(accumulated.display().to_string()) } else { SafeOpenError::Io(e) } })?; + if entry_meta.file_type().is_symlink() { + return Err(SafeOpenError::SymlinkDetected(name.to_string())); + } if !entry_meta.is_dir() { return Err(SafeOpenError::NotFound(format!( "component is not a directory: {}", @@ -548,6 +566,18 @@ mod tests { assert!(safe_open_relative(dir.path(), "src/main.rs", &config).is_ok()); } + #[test] + fn safe_open_root_must_be_directory() { + let dir = tempfile::tempdir().unwrap(); + let root_file = dir.path().join("root.txt"); + std::fs::write(&root_file, "not a directory").unwrap(); + let config = default_config(); + assert!(matches!( + safe_open_relative(&root_file, "test.txt", &config), + Err(SafeOpenError::RootOpenFailed(_)) + )); + } + #[test] fn safe_open_symlink_rejected_when_no_follow() { let dir = tempfile::tempdir().unwrap(); @@ -562,6 +592,41 @@ mod tests { Err(SafeOpenError::SymlinkDetected(_)) )); } + #[cfg(windows)] + { + std::os::windows::fs::symlink_file( + dir.path().join("target.txt"), + dir.path().join("link.txt"), + ) + .unwrap(); + let config = default_config(); + assert!(matches!( + safe_open_relative(dir.path(), "link.txt", &config), + Err(SafeOpenError::SymlinkDetected(_)) + )); + } + } + + #[test] + #[cfg(any(unix, windows))] + fn safe_open_symlink_intermediate_rejected_when_no_follow() { + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("leaf.txt"), "data").unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink(outside.path(), dir.path().join("linkdir")).unwrap(); + #[cfg(windows)] + std::os::windows::fs::symlink_dir(outside.path(), dir.path().join("linkdir")).unwrap(); + let config = default_config(); + let result = safe_open_relative(dir.path(), "linkdir/leaf.txt", &config); + #[cfg(windows)] + assert!(matches!(result, Err(SafeOpenError::SymlinkDetected(_)))); + #[cfg(unix)] + assert!( + result.is_err(), + "opened through symlink: size={:?}", + result.ok().map(|sf| sf.size) + ); } #[test] diff --git a/src/sdk.rs b/src/sdk.rs index e18ebde3..e65a5bd3 100644 --- a/src/sdk.rs +++ b/src/sdk.rs @@ -247,6 +247,76 @@ mod tests { } } + struct BlockingEngine { + entered: Arc, + dropped: Arc, + entered_notify: Arc, + release: Arc, + } + + struct BlockingGuard { + dropped: Arc, + } + + impl Drop for BlockingGuard { + fn drop(&mut self) { + self.dropped.fetch_add(1, Ordering::SeqCst); + } + } + + impl SearchEngine for BlockingEngine { + fn name(&self) -> &'static str { + "duckduckgo" + } + + fn search<'a>( + &'a self, + _request: &'a EngineSearchRequest, + ) -> BoxFuture<'a, Result, EngineError>> { + let entered = self.entered.clone(); + let dropped = self.dropped.clone(); + let entered_notify = self.entered_notify.clone(); + let release = self.release.clone(); + Box::pin(async move { + let _guard = BlockingGuard { dropped }; + entered.fetch_add(1, Ordering::SeqCst); + entered_notify.notify_waiters(); + release.notified().await; + Ok(Vec::new()) + }) + } + } + + fn blocking_client() -> ( + EggsearchClient, + Arc, + Arc, + Arc, + Arc, + ) { + let entered = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicUsize::new(0)); + let entered_notify = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let mut config = AppConfig::default(); + config.search.mode = Mode::Live; + config.local.enabled = false; + let adapter = MetadataSearchAdapter::from_engines( + vec![Arc::new(BlockingEngine { + entered: entered.clone(), + dropped: dropped.clone(), + entered_notify: entered_notify.clone(), + release: release.clone(), + })], + Duration::from_secs(2), + ); + let state = ServerState::with_adapter(config, Arc::new(adapter)); + let client = EggsearchClient { + state: Arc::new(state), + }; + (client, entered, dropped, entered_notify, release) + } + async fn assert_parity( client: &EggsearchClient, name: &str, @@ -385,6 +455,73 @@ mod tests { .await; } + #[tokio::test] + async fn dropping_sdk_execute_aborts_blocked_engine() { + let (client, entered, dropped, entered_notify, _release) = blocking_client(); + let handle = tokio::spawn(async move { + client + .execute( + "web_search", + serde_json::json!({"query": "sdk cancellation"}), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), entered_notify.notified()) + .await + .unwrap(); + assert_eq!(entered.load(Ordering::SeqCst), 1); + handle.abort(); + let _ = tokio::time::timeout(Duration::from_secs(5), handle).await; + tokio::time::timeout(Duration::from_secs(5), async { + while dropped.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn close_token_select_drops_blocked_engine() { + let (client, entered, dropped, entered_notify, _release) = blocking_client(); + let token = tokio_util::sync::CancellationToken::new(); + let canceller = token.clone(); + let runner = tokio::spawn(async move { + tokio::select! { + _ = canceller.cancelled() => Err(SdkError::new( + "client_closed", + "Eggsearch client closed while the operation was running", + None, + None, + )), + result = client.execute( + "web_search", + serde_json::json!({"query": "sdk close race"}), + ) => result, + } + }); + tokio::time::timeout(Duration::from_secs(5), entered_notify.notified()) + .await + .unwrap(); + assert_eq!(entered.load(Ordering::SeqCst), 1); + token.cancel(); + let outcome = tokio::time::timeout(Duration::from_secs(5), runner) + .await + .unwrap() + .unwrap(); + let error = outcome.unwrap_err(); + assert_eq!(error.code, "client_closed"); + tokio::time::timeout(Duration::from_secs(5), async { + while dropped.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + } + #[tokio::test] async fn mode_off_repo_search_through_sdk_keeps_remote_dispatch_at_zero() { let root = TempDir::new().unwrap();