From 91b410f186d3aadfa20f3502e26e50c8d2d46d0a Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Sun, 4 Oct 2026 14:23:31 +0100 Subject: [PATCH 1/4] feat: capture bounded exact-wheel release diagnostics --- .github/scripts/release_diagnostic.py | 267 +++++++++++++++++++++++ .github/workflows/release-diagnostic.yml | 79 +++++++ diagnostics/release-37199991757.json | 139 ++++++++++++ docs/release_validation.md | 28 +++ tests/test_release_diagnostic.py | 145 ++++++++++++ 5 files changed, 658 insertions(+) create mode 100644 .github/scripts/release_diagnostic.py create mode 100644 .github/workflows/release-diagnostic.yml create mode 100644 diagnostics/release-37199991757.json create mode 100644 tests/test_release_diagnostic.py diff --git a/.github/scripts/release_diagnostic.py b/.github/scripts/release_diagnostic.py new file mode 100644 index 0000000..568df29 --- /dev/null +++ b/.github/scripts/release_diagnostic.py @@ -0,0 +1,267 @@ +"""Bounded wheel-only investigation; never emits release-validation evidence.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import importlib.util +import json +import os +from pathlib import Path +import platform +import re +import signal +import subprocess +import sys +import time + + +LIBRARIES = ("autonerves", "autofit", "autoarray", "autogalaxy", "autolens") + + +def load_manifest(path): + data = json.loads(Path(path).read_text()) + if data.get("schema_version") != 1: + raise ValueError("unsupported diagnostic manifest") + for key in ("workspace_sha", "hands_sha"): + if not re.fullmatch(r"[0-9a-f]{40}", data.get(key, "")): + raise ValueError(f"{key} must be an immutable commit") + if not re.fullmatch(r"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+", data.get("workspace_repo", "")): + raise ValueError("invalid workspace repository") + if not re.fullmatch(r"3\.\d+\.\d+", data.get("python", "")): + raise ValueError("pin the Python patch version") + script = Path(data.get("script", "")) + if script.is_absolute() or ".." in script.parts or script.suffix != ".py": + raise ValueError("script must be a relative Python path inside scripts/") + packages = data.get("packages", {}) + if not all(name in packages for name in (*LIBRARIES, "jax", "jaxlib")): + raise ValueError("missing library/runtime package pins") + for name, version in packages.items(): + if not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch( + r"[0-9][A-Za-z0-9_.+!-]*", version + ): + raise ValueError("package pins must be names and exact versions") + return data + + +def prepare(manifest, output, github_output=None): + output.mkdir(parents=True, exist_ok=True) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + (output / "requirements.txt").write_text( + "".join( + f"{name}=={version}\n" + for name, version in sorted(manifest["packages"].items()) + ) + ) + if github_output: + with Path(github_output).open("a") as stream: + for key in ("python", "workspace_repo", "workspace_sha", "hands_sha"): + print(f"{key}={manifest[key]}", file=stream) + + +def provenance(manifest, workspace, hands): + errors = [] + receipt = { + "diagnostic_only": True, + "python": platform.python_version(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "affinity": sorted(os.sched_getaffinity(0)), + "packages": {}, + "imports": {}, + } + receipt["runner_image"] = { + key: os.environ.get(key) for key in ("ImageOS", "ImageVersion") + } + receipt["cpu_quota"] = {} + for path in ("/sys/fs/cgroup/cpu.max", "/sys/fs/cgroup/cpuset.cpus.effective"): + try: + receipt["cpu_quota"][path] = Path(path).read_text().strip() + except OSError: + receipt["cpu_quota"][path] = None + for label, root, expected in ( + ("workspace", workspace, manifest["workspace_sha"]), + ("hands", hands, manifest["hands_sha"]), + ): + actual = subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip() + receipt[label + "_sha"] = actual + if actual != expected: + errors.append(f"{label} commit differs from manifest") + if receipt["python"] != manifest["python"]: + errors.append("Python patch version differs from manifest") + for name, expected in manifest["packages"].items(): + try: + actual = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + actual = None + receipt["packages"][name] = actual + if actual != expected: + errors.append(f"{name}: expected {expected}, installed {actual}") + for name in (*LIBRARIES, "jax", "jaxlib"): + spec = importlib.util.find_spec(name) + origin = spec.origin if spec else None + receipt["imports"][name] = origin + if not origin or "site-packages" not in Path(origin).parts: + errors.append(f"{name} is not imported from installed wheels") + receipt["errors"] = errors + return receipt + + +def native_dump(pid, output): + """Collect while the child lives; each helper has its own short deadline.""" + blocks = [] + for task in sorted(Path(f"/proc/{pid}/task").glob("*")): + try: + blocks.append( + f"{task.name}: {(task / 'comm').read_text().strip()} " + f"{(task / 'wchan').read_text().strip()}" + ) + except OSError as error: + blocks.append(str(error)) + commands = [ + ["py-spy", "dump", "--native", "--pid", str(pid)], + [ + "gdb", + "--batch", + "-p", + str(pid), + "-ex", + "set pagination off", + "-ex", + "thread apply all bt", + "-ex", + "detach", + ], + ] + for command in commands: + blocks.append("$ " + " ".join(command)) + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=20) + blocks.extend([f"exit={result.returncode}", result.stdout, result.stderr]) + except (OSError, subprocess.TimeoutExpired) as error: + blocks.append(str(error)) + if isinstance(error, subprocess.TimeoutExpired): + for partial in (error.stdout, error.stderr): + if partial: + blocks.append( + partial.decode(errors="replace") + if isinstance(partial, bytes) + else partial + ) + output.write_text("\n".join(blocks)) + + +def trial(command, cwd, env, output, *, cap=300, dump_after=120, dump=native_dump): + started = time.monotonic() + timed_out = False + with output.open("w") as log: + proc = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + try: + proc.wait(timeout=dump_after) + except subprocess.TimeoutExpired: + dump(proc.pid, output.with_suffix(".native.txt")) + try: + proc.wait(timeout=max(0, cap - (time.monotonic() - started))) + except subprocess.TimeoutExpired: + timed_out = True + finally: + # Also retire any simulator descendants; none may survive this trial. + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + return { + "status": ( + "timeout" if timed_out else "pass" if proc.returncode == 0 else "fail" + ), + "returncode": proc.returncode, + "elapsed_s": round(time.monotonic() - started, 3), + "cap_s": cap, + "log": output.name, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("prepare", "run")) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--github-output") + parser.add_argument("--workspace", type=Path) + parser.add_argument("--hands", type=Path) + parser.add_argument("--repeats", type=int, default=6) + args = parser.parse_args(argv) + if not 1 <= args.repeats <= 6: + parser.error("repeats must be between 1 and 6") + manifest = load_manifest(args.manifest) + output = args.output.resolve() + prepare(manifest, output, args.github_output) + if args.mode == "prepare": + return 0 + if not args.workspace or not args.hands: + parser.error("run needs --workspace and --hands") + workspace, hands = args.workspace.resolve(), args.hands.resolve() + receipt = provenance(manifest, workspace, hands) + (output / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") + if receipt["errors"]: + print("Provenance mismatch: " + "; ".join(receipt["errors"]), flush=True) + return 2 + # Import only the pinned Hands resolver into this parent. Child imports + # never inherit source directories from sys.path. + sys.path.insert(0, str(hands / "autohands")) + from env_config import build_env_for_script, load_env_config + + script = workspace / "scripts" / manifest["script"] + if not script.is_file() or not script.resolve().is_relative_to( + workspace / "scripts" + ): + raise ValueError("script missing or outside workspace") + config = load_env_config(workspace / "config/build/profile_release.yaml") + env = build_env_for_script(script, config) + env.pop("PYTHONPATH", None) + env.update(PYTHONUNBUFFERED="1", JAX_TRACEBACK_FILTERING="off") + receipt["environment"] = { + key: value + for key, value in env.items() + if key.startswith(("PYAUTO_", "JAX_", "XLA_", "OMP_", "MKL_", "OPENBLAS_")) + } + (output / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") + results = { + "diagnostic_only": True, + "source_run": manifest["source_run"], + "trials": [], + } + for index in range(args.repeats): + result = trial( + [sys.executable, str(script)], + workspace, + env, + output / f"trial-{index + 1}.log", + ) + results["trials"].append(result) + (output / "diagnostic-results.json").write_text( + json.dumps(results, indent=2) + "\n" + ) + print(json.dumps(result), flush=True) + if result["status"] != "pass": + return 1 + print( + "Diagnostic repetitions passed; this is not release-validation clearance.", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/release-diagnostic.yml b/.github/workflows/release-diagnostic.yml new file mode 100644 index 0000000..60443b1 --- /dev/null +++ b/.github/workflows/release-diagnostic.yml @@ -0,0 +1,79 @@ +name: Release Diagnostic (not validation) + +on: + pull_request: + paths: + - .github/workflows/release-diagnostic.yml + - .github/scripts/release_diagnostic.py + - diagnostics/release-37199991757.json + workflow_dispatch: + inputs: + repeats: + description: "Fresh target processes (1–6); stop on first failure" + type: number + default: 6 + +permissions: + contents: read + +jobs: + diagnose: + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + PYTHONPATH: "" + MANIFEST: diagnostics/release-37199991757.json + steps: + - uses: actions/checkout@v4 + - name: Validate immutable manifest and write requirements + id: manifest + run: >- + python3 .github/scripts/release_diagnostic.py prepare + --manifest "$MANIFEST" --output diagnostic-results + --github-output "$GITHUB_OUTPUT" + - uses: actions/checkout@v4 + with: + repository: ${{ steps.manifest.outputs.workspace_repo }} + ref: ${{ steps.manifest.outputs.workspace_sha }} + path: diagnostic-workspace + persist-credentials: false + - uses: actions/checkout@v4 + with: + repository: PyAutoLabs/PyAutoHands + ref: ${{ steps.manifest.outputs.hands_sha }} + path: diagnostic-hands + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ steps.manifest.outputs.python }} + - name: Install recovered wheel environment and native inspection tools + run: | + python -m venv diagnostic-env + diagnostic-env/bin/python -m pip install \ + --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple \ + --report diagnostic-results/pip-install.json \ + -r diagnostic-results/requirements.txt + diagnostic-env/bin/python -m pip install py-spy + echo "$PWD/diagnostic-env/bin" >> "$GITHUB_PATH" + sudo apt-get update -qq + sudo apt-get install -y -qq gdb + # Ephemeral runner only; allow sibling debuggers to attach to Python. + sudo sysctl -w kernel.yama.ptrace_scope=0 + - name: Bounded release-profile repetitions against installed wheels + env: + REPEATS: ${{ inputs.repeats || 6 }} + run: >- + diagnostic-env/bin/python .github/scripts/release_diagnostic.py run + --manifest "$MANIFEST" --output diagnostic-results + --workspace diagnostic-workspace --hands diagnostic-hands + --repeats "$REPEATS" + - name: Preserve diagnostic evidence, including failure and native stacks + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-diagnostic-${{ github.run_id }} + path: | + diagnostic-results/ + diagnostic-workspace/dataset/multi_dataset/lens_sersic/*.fits + if-no-files-found: error diff --git a/diagnostics/release-37199991757.json b/diagnostics/release-37199991757.json new file mode 100644 index 0000000..38b7147 --- /dev/null +++ b/diagnostics/release-37199991757.json @@ -0,0 +1,139 @@ +{ + "hands_sha": "760affdbcbf0605266e49b33a65f7bef89f9b0b3", + "library_shas": { + "PyAutoArray": "1cbabf9a44bfb98925e35954c263d393b7affc9d", + "PyAutoFit": "035ef1c8f79e4d488bb842c926e7d75c976cf8b6", + "PyAutoGalaxy": "4675fb265dd4c6d40aea8f809285b6d14dfb6325", + "PyAutoLens": "f9cff6b5c918748de131aade02aa864f17bfa0eb", + "PyAutoNerves": "d7e9f467da187e885139a5adb16baefbc8df59a7" + }, + "limitations": [ + "Package versions recovered from successful pip installation log lines; original wheel hashes and runner native stacks were not recorded.", + "The original runner OS image and CPU hardware are not pinned. Record the diagnostic topology; passing repetitions do not clear release validation.", + "Dataset is regenerated by the unchanged workspace bootstrap; original generated FITS files were not retained in the failed artifact." + ], + "packages": { + "MarkupSafe": "3.0.4", + "PyYAML": "6.0.3", + "SQLAlchemy": "2.0.54", + "absl-py": "2.5.0", + "alabaster": "1.0.0", + "anesthetic": "2.16.0", + "array_api_compat": "1.15.0", + "astropy": "8.0.1", + "astropy-iers-data": "0.2026.9.28.0.59.37", + "asttokens": "3.0.2", + "astunparse": "1.6.3", + "attrs": "26.1.0", + "autoarray": "2026.10.4.1.dev80201", + "autofit": "2026.10.4.1.dev80201", + "autogalaxy": "2026.10.4.1.dev80201", + "autolens": "2026.10.4.1.dev80201", + "autonerves": "2026.10.4.1.dev80201", + "babel": "2.18.0", + "blackjax": "1.7.1", + "certifi": "2026.7.22", + "charset_normalizer": "3.5.2", + "cloudpickle": "3.1.2", + "contourpy": "1.4.0", + "coolest": "0.1.11", + "corner": "2.2.2", + "cycler": "0.12.1", + "decorator": "5.3.1", + "dill": "0.4.1", + "dm-tree": "0.1.10", + "docutils": "0.22.4", + "dynesty": "2.1.5", + "emcee": "3.1.6", + "executing": "2.2.1", + "fonttools": "4.66.1", + "gast": "0.7.0", + "getdist": "1.4", + "gprof2dot": "2021.2.21", + "greenlet": "3.5.6", + "h5py": "3.16.0", + "idna": "3.20", + "imageio": "2.38.0", + "imagesize": "2.0.1", + "iniconfig": "2.3.0", + "ipython": "9.12.0", + "ipython-pygments-lexers": "1.1.1", + "jax": "0.10.2", + "jax_zero_contour": "2.0.0", + "jaxlib": "0.10.2", + "jaxnnls": "1.0.1", + "jedi": "0.20.0", + "jinja2": "3.1.6", + "joblib": "1.6.0", + "jsonpickle": "4.1.3", + "kiwisolver": "1.5.1", + "lazy-loader": "0.6", + "llvmlite": "0.50.0", + "matplotlib": "3.11.2", + "matplotlib-inline": "0.2.2", + "ml_dtypes": "0.6.0", + "mypy-extensions": "1.1.0", + "nautilus-sampler": "1.0.5", + "networkx": "3.1", + "nufftax": "0.6.1", + "numba": "0.68.0", + "numpy": "2.5.3", + "numpydoc": "1.11.0", + "opt_einsum": "3.4.0", + "optax": "0.2.8", + "packaging": "26.3", + "pandas": "3.0.6", + "parso": "0.8.7", + "pexpect": "4.9.0", + "pillow": "12.3.0", + "pluggy": "1.6.0", + "prompt_toolkit": "3.0.53", + "psutil": "6.1.0", + "ptyprocess": "0.7.0", + "pure-eval": "0.2.4", + "pyerfa": "2.0.1.5", + "pygments": "2.21.0", + "pyparsing": "3.3.3", + "pytest": "9.1.1", + "python-dateutil": "3.9.0", + "pyvis": "0.3.2", + "requests": "2.34.2", + "roman-numerals": "4.1.0", + "scikit-image": "0.26.0", + "scikit-learn": "1.8.0", + "scipy": "1.17.1", + "seaborn": "0.13.2", + "setuptools": "84.0.0", + "six": "1.17.0", + "snowballstemmer": "3.1.1", + "sphinx": "9.1.0", + "sphinxcontrib-applehelp": "2.0.0", + "sphinxcontrib-devhelp": "2.0.0", + "sphinxcontrib-htmlhelp": "2.1.0", + "sphinxcontrib-jsmath": "1.0.1", + "sphinxcontrib-qthelp": "2.0.0", + "sphinxcontrib-serializinghtml": "2.0.0", + "stack_data": "0.6.3", + "tfp-nightly": "0.26.0.dev20260713", + "threadpoolctl": "3.7.0", + "tifffile": "2026.9.20", + "timeout-decorator": "0.5.0", + "tqdm": "4.70.1", + "traitlets": "5.16.1", + "typing-extensions": "4.16.0", + "typing-inspect": "0.9.0", + "urllib3": "2.8.0", + "wcwidth": "0.9.1", + "wheel": "0.48.0", + "wrapt": "2.5.0", + "xxhash": "3.4.1", + "zeus-mcmc": "2.5.4" + }, + "python": "3.12.14", + "schema_version": 1, + "script": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "source_job": "integrate / run_scripts (3.12, autolens_test, multi_dataset)", + "source_run": "https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37199991757", + "workspace_repo": "PyAutoLabs/autolens_workspace_test", + "workspace_sha": "df35fd4553f861575af7c27ad25b5f4d8d97d09c" +} diff --git a/docs/release_validation.md b/docs/release_validation.md index ab600f5..cbff2ac 100644 --- a/docs/release_validation.md +++ b/docs/release_validation.md @@ -195,3 +195,31 @@ dispatching the `release-integrate.yml` channel), an ingested rehearsal-only report still (correctly) gates YELLOW: the source was built and TestPyPI-installed, but not yet exercised at release fidelity. `mode: release` is what supplies the `integrate` stage that flips this to GREEN-eligible. +# Bounded investigation of a failed wheel run + +`release-diagnostic.yml` investigates the saved `rectangular_rtu.py` timeout +from run `37199991757`. It runs automatically only on PRs changing that +diagnostic's workflow, runner or manifest, and can also be dispatched manually +with `repeats` between 1 and 6. It does not run the integration matrix. + +The manifest `diagnostics/release-37199991757.json` preserves the final 115 +package versions from the failed installation log, the exact workspace and +Hands commits, Python 3.12.14 and the rehearsed library SHAs. In particular, +the failed release workflow downgraded JAX/JAXlib to 0.10.2, while the earlier +passing source retimes used 0.11.2. This difference is a confound to test, +not a demonstrated cause. + +The runner resolves the pinned workspace's release profile through pinned +Hands, verifies package versions and wheel import origins, and executes only +the affected script in fresh processes. It captures `/proc` thread states, +`py-spy --native` and GDB stacks at 120 seconds, terminates at 300 seconds, +and stops on the first failure. Diagnostic artifacts preserve the manifest, +pip installation URLs/hashes, actual runtime and environment provenance, +stdout/stderr, native capture failures, results and newly generated FITS data. +The original runner image, hardware, wheel hashes and generated FITS data +were not retained; the receipt exposes these reproduction limits. + +These are **diagnostic results only**. Neither passing repetitions nor a green +workflow clears the failed release. This workflow emits no release stage +report and never calls validation ingest. Do not raise caps, quarantine the +script, or identify a causal repair without reproducible or native evidence. diff --git a/tests/test_release_diagnostic.py b/tests/test_release_diagnostic.py new file mode 100644 index 0000000..3246a7c --- /dev/null +++ b/tests/test_release_diagnostic.py @@ -0,0 +1,145 @@ +"""The diagnostic must fail closed and capture the live child before killing it.""" + +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location( + "release_diagnostic", ROOT / ".github/scripts/release_diagnostic.py" +) +diag = importlib.util.module_from_spec(spec) +spec.loader.exec_module(diag) + + +def manifest(): + return diag.load_manifest(ROOT / "diagnostics/release-37199991757.json") + + +@pytest.mark.parametrize( + "key,value", + [ + ("workspace_sha", "main"), + ("hands_sha", "../bad"), + ("python", "3.12"), + ("script", "../../outside.py"), + ("script", "/tmp/outside.py"), + ("workspace_repo", "../repo"), + ("packages", {}), + ("schema_version", 2), + ], +) +def test_rejects_unpinned_or_escaping_manifest(tmp_path, key, value): + data = manifest() + data[key] = value + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data)) + with pytest.raises(ValueError): + diag.load_manifest(path) + + +def test_prepared_pins_retain_failed_runtime_and_no_release_artifact(tmp_path): + data = manifest() + diag.prepare(data, tmp_path) + requirements = (tmp_path / "requirements.txt").read_text() + assert "jax==0.10.2\n" in requirements + assert "jaxlib==0.10.2\n" in requirements + assert len(requirements.splitlines()) == 115 + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "manifest.json", + "requirements.txt", + ] + + +def test_runtime_and_source_shadowing_fail_provenance(monkeypatch, tmp_path): + data = manifest() + monkeypatch.setattr(diag.subprocess, "check_output", lambda *a, **kw: "0" * 40) + monkeypatch.setattr(diag.platform, "python_version", lambda: "3.12.10") + monkeypatch.setattr( + diag.importlib.metadata, "version", lambda name: data["packages"][name] + ) + monkeypatch.setattr( + diag.importlib.util, + "find_spec", + lambda name: type("Spec", (), {"origin": f"/source/{name}/__init__.py"})(), + ) + receipt = diag.provenance(data, tmp_path, tmp_path) + assert receipt["diagnostic_only"] + assert "Python patch version differs from manifest" in receipt["errors"] + assert "workspace commit differs from manifest" in receipt["errors"] + assert any("autolens is not imported" in error for error in receipt["errors"]) + + +def test_live_dump_precedes_timeout_and_child_is_reaped(tmp_path): + captured = [] + + def dump(pid, output): + os.kill(pid, 0) # must still exist when sampled + captured.append(pid) + output.write_text("live stack witness") + + result = diag.trial( + [ + sys.executable, + "-c", + "import time; print('started', flush=True); time.sleep(20)", + ], + tmp_path, + os.environ.copy(), + tmp_path / "trial.log", + cap=0.3, + dump_after=0.1, + dump=dump, + ) + assert result["status"] == "timeout" + assert result["elapsed_s"] < 3 + assert "started" in (tmp_path / "trial.log").read_text() + assert (tmp_path / "trial.native.txt").read_text() == "live stack witness" + with pytest.raises(ProcessLookupError): + os.kill(captured[0], 0) + + +def test_failure_exit_and_output_are_not_reported_as_pass(tmp_path): + result = diag.trial( + [sys.executable, "-c", "print('adverse evidence'); raise SystemExit(7)"], + tmp_path, + os.environ.copy(), + tmp_path / "failed.log", + cap=3, + dump_after=1, + ) + assert result["status"] == "fail" + assert result["returncode"] == 7 + assert "adverse evidence" in (tmp_path / "failed.log").read_text() + + +def test_native_helper_failure_is_preserved(monkeypatch, tmp_path): + def unavailable(*args, **kwargs): + raise FileNotFoundError("diagnostic helper unavailable") + + monkeypatch.setattr(subprocess, "run", unavailable) + output = tmp_path / "native.txt" + diag.native_dump(os.getpid(), output) + assert output.read_text().count("diagnostic helper unavailable") == 2 + + +def test_repeats_cannot_exceed_bounded_budget(tmp_path): + with pytest.raises(SystemExit) as exc: + diag.main( + [ + "prepare", + "--manifest", + str(ROOT / "diagnostics/release-37199991757.json"), + "--output", + str(tmp_path), + "--repeats", + "7", + ] + ) + assert exc.value.code == 2 From a954d5a2156ae54e6dc58bd72c91c16629b9bf8a Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Sun, 4 Oct 2026 14:44:52 +0100 Subject: [PATCH 2/4] test: compare JAX runtimes against captured Cholesky deadlock --- .github/scripts/release_diagnostic.py | 116 ++++++++++++++++++- .github/workflows/release-diagnostic.yml | 15 ++- diagnostics/release-37199991757-native.txt | 43 +++++++ diagnostics/release-37199991757.json | 1 + docs/release_validation.md | 13 ++- tests/test_release_diagnostic.py | 124 +++++++++++++++++++++ 6 files changed, 305 insertions(+), 7 deletions(-) create mode 100644 diagnostics/release-37199991757-native.txt diff --git a/.github/scripts/release_diagnostic.py b/.github/scripts/release_diagnostic.py index 568df29..bed1bf9 100644 --- a/.github/scripts/release_diagnostic.py +++ b/.github/scripts/release_diagnostic.py @@ -26,7 +26,9 @@ def load_manifest(path): for key in ("workspace_sha", "hands_sha"): if not re.fullmatch(r"[0-9a-f]{40}", data.get(key, "")): raise ValueError(f"{key} must be an immutable commit") - if not re.fullmatch(r"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+", data.get("workspace_repo", "")): + if not re.fullmatch( + r"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+", data.get("workspace_repo", "") + ): raise ValueError("invalid workspace repository") if not re.fullmatch(r"3\.\d+\.\d+", data.get("python", "")): raise ValueError("pin the Python patch version") @@ -41,6 +43,10 @@ def load_manifest(path): r"[0-9][A-Za-z0-9_.+!-]*", version ): raise ValueError("package pins must be names and exact versions") + if "candidate_jax" in data and not re.fullmatch( + r"[0-9]+\.[0-9]+\.[0-9]+", data["candidate_jax"] + ): + raise ValueError("candidate JAX must be an exact release version") return data @@ -192,25 +198,131 @@ def trial(command, cwd, env, output, *, cap=300, dump_after=120, dump=native_dum } +def comparison_verdict(rows, expected): + if not rows or expected % 2: + return "incomplete" + order = ["control", "candidate"] * (expected // 2) + if len(rows) > expected or [row["arm"] for row in rows] != order[: len(rows)]: + return "invalid-order" + if any(row["status"] != "pass" for row in rows if row["arm"] == "candidate"): + return "candidate-failed" + if any(row["status"] not in {"pass", "timeout"} for row in rows): + return "control-failed" + if len(rows) != expected: + return "incomplete" + if not any( + row.get("cholesky_pool_stall") for row in rows if row["arm"] == "control" + ): + return "inconclusive" + return "captured-stall-avoided" + + +def compare(args, manifest, output): + """ABAB on one runner and dataset; keep adverse controls verbatim.""" + if not args.candidate_python or args.repeats % 2: + raise ValueError("comparison needs candidate Python and 2, 4 or 6 trials") + rows = [] + report = { + "diagnostic_only": True, + "source_run": manifest["source_run"], + "trials": rows, + } + for index in range(args.repeats): + arm = "control" if index % 2 == 0 else "candidate" + directory = output / f"{index + 1}-{arm}" + # Resolving a venv interpreter symlink would escape into base Python. + python = ( + sys.executable + if arm == "control" + else str(args.candidate_python.absolute()) + ) + if directory.exists(): + raise ValueError( + f"preserve previous evidence; choose a fresh output directory: {directory}" + ) + command = [ + python, + str(Path(__file__).resolve()), + "run", + "--manifest", + str(args.manifest.resolve()), + "--output", + str(directory), + "--workspace", + str(args.workspace.resolve()), + "--hands", + str(args.hands.resolve()), + "--repeats", + "1", + ] + if arm == "candidate": + command.append("--candidate") + completed = subprocess.run(command, timeout=360) + result_path = directory / "diagnostic-results.json" + if not result_path.exists(): + raise RuntimeError( + f"{arm} failed before a measured trial (exit {completed.returncode})" + ) + result = json.loads(result_path.read_text())["trials"][0] + expected_exit = 0 if result["status"] == "pass" else 1 + if completed.returncode != expected_exit: + raise RuntimeError(f"{arm} runner exit disagrees with its trial artifact") + result["arm"] = arm + result["directory"] = directory.name + native_path = directory / "trial-1.native.txt" + native = native_path.read_text() if native_path.exists() else "" + result["cholesky_pool_stall"] = result["status"] == "timeout" and all( + native.count(marker) >= 4 + for marker in ( + "BlockingCounter::Wait", + "ParallelBatchMap", + "CholeskyFactorization", + ) + ) + rows.append(result) + report["verdict"] = comparison_verdict(rows, args.repeats) + (output / "comparison.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(result), flush=True) + if result["status"] not in {"pass", "timeout"} or ( + arm == "candidate" and result["status"] != "pass" + ): + return 1 + print( + "Diagnostic comparison: " + report["verdict"] + "; not release clearance", + flush=True, + ) + return 0 if report["verdict"] == "captured-stall-avoided" else 2 + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("mode", choices=("prepare", "run")) + parser.add_argument("mode", choices=("prepare", "run", "compare")) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--github-output") parser.add_argument("--workspace", type=Path) parser.add_argument("--hands", type=Path) parser.add_argument("--repeats", type=int, default=6) + parser.add_argument("--candidate", action="store_true") + parser.add_argument("--candidate-python", type=Path) args = parser.parse_args(argv) if not 1 <= args.repeats <= 6: parser.error("repeats must be between 1 and 6") manifest = load_manifest(args.manifest) + if args.candidate: + manifest["packages"].update( + jax=manifest["candidate_jax"], jaxlib=manifest["candidate_jax"] + ) output = args.output.resolve() prepare(manifest, output, args.github_output) if args.mode == "prepare": return 0 if not args.workspace or not args.hands: parser.error("run needs --workspace and --hands") + if args.mode == "compare": + if args.candidate: + parser.error("comparison must start from the unmodified control manifest") + return compare(args, manifest, output) workspace, hands = args.workspace.resolve(), args.hands.resolve() receipt = provenance(manifest, workspace, hands) (output / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") diff --git a/.github/workflows/release-diagnostic.yml b/.github/workflows/release-diagnostic.yml index 60443b1..71132f5 100644 --- a/.github/workflows/release-diagnostic.yml +++ b/.github/workflows/release-diagnostic.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: inputs: repeats: - description: "Fresh target processes (1–6); stop on first failure" + description: "Alternating control/candidate processes (2, 4 or 6)" type: number default: 6 @@ -55,18 +55,27 @@ jobs: --report diagnostic-results/pip-install.json \ -r diagnostic-results/requirements.txt diagnostic-env/bin/python -m pip install py-spy + python .github/scripts/release_diagnostic.py prepare --candidate \ + --manifest "$MANIFEST" --output diagnostic-results/candidate-install + python -m venv diagnostic-candidate + diagnostic-candidate/bin/python -m pip install \ + --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple \ + --report diagnostic-results/candidate-install/pip-install.json \ + -r diagnostic-results/candidate-install/requirements.txt echo "$PWD/diagnostic-env/bin" >> "$GITHUB_PATH" sudo apt-get update -qq sudo apt-get install -y -qq gdb # Ephemeral runner only; allow sibling debuggers to attach to Python. sudo sysctl -w kernel.yama.ptrace_scope=0 - - name: Bounded release-profile repetitions against installed wheels + - name: Interleaved runtime comparison against the same release wheels env: REPEATS: ${{ inputs.repeats || 6 }} run: >- - diagnostic-env/bin/python .github/scripts/release_diagnostic.py run + diagnostic-env/bin/python .github/scripts/release_diagnostic.py compare --manifest "$MANIFEST" --output diagnostic-results --workspace diagnostic-workspace --hands diagnostic-hands + --candidate-python diagnostic-candidate/bin/python --repeats "$REPEATS" - name: Preserve diagnostic evidence, including failure and native stacks if: always() diff --git a/diagnostics/release-37199991757-native.txt b/diagnostics/release-37199991757-native.txt new file mode 100644 index 0000000..aebcfde --- /dev/null +++ b/diagnostics/release-37199991757-native.txt @@ -0,0 +1,43 @@ +# Native witness: release diagnostic 37205459198 + +Source: https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37205459198 +Artifact: release-diagnostic-37205459198 / diagnostic-results/trial-2.native.txt +Raw native file SHA256: 1a025a43d895e247d7052ef0c2d84dfc33dafdd8fe92a4f1df726d39c42ced53 + +Trial1 PASS9.905s; trial2 TIMEOUT300.021s after2.6s compile. Both native tools exited0. +Python3.12.14, JAX/JAXlib0.10.2, four CPUs, XLA_FLAGS=--xla_cpu_multi_thread_eigen=false. +Four Eigen workers wait in LAPACK Cholesky batch fan-out; no FftThunk or ducc0 frames. + +Below: thread headings and selected frames, signatures and runner paths trimmed. + +Thread 28 (Thread 0x7ff0f93536c0 (LWP 3258) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#23 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) + +Thread 27 (Thread 0x7ff0f8b526c0 (LWP 3259) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#23 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) + +Thread 26 (Thread 0x7ff0f3fff6c0 (LWP 3260) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#24 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) + +Thread 25 (Thread 0x7ff0f37fe6c0 (LWP 3261) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#23 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) diff --git a/diagnostics/release-37199991757.json b/diagnostics/release-37199991757.json index 38b7147..a1d7855 100644 --- a/diagnostics/release-37199991757.json +++ b/diagnostics/release-37199991757.json @@ -1,4 +1,5 @@ { + "candidate_jax": "0.11.2", "hands_sha": "760affdbcbf0605266e49b33a65f7bef89f9b0b3", "library_shas": { "PyAutoArray": "1cbabf9a44bfb98925e35954c263d393b7affc9d", diff --git a/docs/release_validation.md b/docs/release_validation.md index cbff2ac..439d87c 100644 --- a/docs/release_validation.md +++ b/docs/release_validation.md @@ -200,7 +200,7 @@ is what supplies the `integrate` stage that flips this to GREEN-eligible. `release-diagnostic.yml` investigates the saved `rectangular_rtu.py` timeout from run `37199991757`. It runs automatically only on PRs changing that diagnostic's workflow, runner or manifest, and can also be dispatched manually -with `repeats` between 1 and 6. It does not run the integration matrix. +with `repeats` of 2, 4 or 6 total trials. It does not run the integration matrix. The manifest `diagnostics/release-37199991757.json` preserves the final 115 package versions from the failed installation log, the exact workspace and @@ -213,7 +213,12 @@ The runner resolves the pinned workspace's release profile through pinned Hands, verifies package versions and wheel import origins, and executes only the affected script in fresh processes. It captures `/proc` thread states, `py-spy --native` and GDB stacks at 120 seconds, terminates at 300 seconds, -and stops on the first failure. Diagnostic artifacts preserve the manifest, +and stops on any candidate failure. The comparison interleaves original 0.10.2 +controls and 0.11.2 candidates on the same runner and dataset, changing only +JAX/JAXlib in separate virtual environments. Original control timeouts remain +in the artifacts; a successful comparison requires a native-confirmed Cholesky +pool stall in at least one control and every candidate passing. Passing controls +alone are inconclusive, not evidence of a remedy. Diagnostic artifacts preserve the manifest, pip installation URLs/hashes, actual runtime and environment provenance, stdout/stderr, native capture failures, results and newly generated FITS data. The original runner image, hardware, wheel hashes and generated FITS data @@ -223,3 +228,7 @@ These are **diagnostic results only**. Neither passing repetitions nor a green workflow clears the failed release. This workflow emits no release stage report and never calls validation ingest. Do not raise caps, quarantine the script, or identify a causal repair without reproducible or native evidence. + +The first exact hosted replay, [37205459198](https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37205459198), passed once in 9.905s and then stalled after 2.6s compilation. Both native tools captured all four Eigen workers waiting in `BlockingCounter::Wait` through `ParallelBatchMap`, `CholeskyFactorization` and `lapack_dpotrf_ffi`. The committed [native witness](../diagnostics/release-37199991757-native.txt) includes the raw artifact's SHA256. There are no FFT/ducc0 frames. + +[JAX 0.10.2](https://github.com/jax-ml/jax/blob/jax-v0.10.2/jaxlib/cpu/lapack_kernels.cc) schedules LAPACK batch chunks to the thread pool and blocks waiting for them; [JAX 0.11.2](https://github.com/jax-ml/jax/blob/jax-v0.11.2/jaxlib/cpu/lapack_kernels.cc) compiles this parallel path out of open-source builds. This is the source-grounded candidate tested by the comparison, distinct from the historical FFT workaround. No upstream report was filed. diff --git a/tests/test_release_diagnostic.py b/tests/test_release_diagnostic.py index 3246a7c..603b465 100644 --- a/tests/test_release_diagnostic.py +++ b/tests/test_release_diagnostic.py @@ -143,3 +143,127 @@ def test_repeats_cannot_exceed_bounded_budget(tmp_path): ] ) assert exc.value.code == 2 + + +@pytest.mark.parametrize( + "control,candidate,signature,expected", + [ + ("timeout", "pass", True, "captured-stall-avoided"), + ("timeout", "pass", False, "inconclusive"), + ("pass", "pass", False, "inconclusive"), + ("timeout", "timeout", True, "candidate-failed"), + ("fail", "pass", False, "control-failed"), + ], +) +def test_comparison_requires_native_control_and_passing_candidate( + control, candidate, signature, expected +): + rows = [ + {"arm": "control", "status": control, "cholesky_pool_stall": signature}, + {"arm": "candidate", "status": candidate}, + ] + assert diag.comparison_verdict(rows, 2) == expected + partial = ( + expected if expected in {"candidate-failed", "control-failed"} else "incomplete" + ) + assert diag.comparison_verdict(rows, 4) == partial + assert diag.comparison_verdict(list(reversed(rows)), 2) == "invalid-order" + + +def test_comparison_interleaves_and_preserves_control_failures_and_venv_path( + tmp_path, monkeypatch +): + from argparse import Namespace + + python = tmp_path / "candidate-python" + python.symlink_to(sys.executable) + args = Namespace( + candidate_python=python, + repeats=6, + manifest=ROOT / "diagnostics/release-37199991757.json", + workspace=tmp_path, + hands=tmp_path, + ) + commands = [] + + def run(command, **kwargs): + commands.append(command) + candidate = "--candidate" in command + directory = Path(command[command.index("--output") + 1]) + directory.mkdir() + result = { + "status": "pass" if candidate else "timeout", + "returncode": 0 if candidate else -9, + } + (directory / "diagnostic-results.json").write_text( + json.dumps({"trials": [result]}) + ) + if not candidate: + (directory / "trial-1.native.txt").write_text( + "BlockingCounter::Wait ParallelBatchMap CholeskyFactorization\n" * 4 + ) + return subprocess.CompletedProcess(command, 0 if candidate else 1) + + monkeypatch.setattr(diag.subprocess, "run", run) + assert diag.compare(args, manifest(), tmp_path) == 0 + assert ["--candidate" in command for command in commands] == [False, True] * 3 + assert all(command[0] == str(python.absolute()) for command in commands[1::2]) + report = json.loads((tmp_path / "comparison.json").read_text()) + assert [row["status"] for row in report["trials"]] == ["timeout", "pass"] * 3 + with pytest.raises(ValueError, match="preserve previous evidence"): + diag.compare(args, manifest(), tmp_path) + + +def test_candidate_preparation_changes_only_the_runtime_pair(tmp_path): + diag.main( + [ + "prepare", + "--candidate", + "--manifest", + str(ROOT / "diagnostics/release-37199991757.json"), + "--output", + str(tmp_path), + ] + ) + actual = json.loads((tmp_path / "manifest.json").read_text()) + original = manifest() + changed = { + key + for key in original["packages"] + if actual["packages"][key] != original["packages"][key] + } + assert changed == {"jax", "jaxlib"} + assert actual["packages"]["jax"] == actual["packages"]["jaxlib"] == "0.11.2" + assert actual["workspace_sha"] == original["workspace_sha"] + + +def test_early_candidate_failure_has_adverse_durable_verdict(tmp_path, monkeypatch): + from argparse import Namespace + + args = Namespace( + candidate_python=Path(sys.executable), + repeats=6, + manifest=ROOT / "diagnostics/release-37199991757.json", + workspace=tmp_path, + hands=tmp_path, + ) + + def run(command, **kwargs): + candidate = "--candidate" in command + directory = Path(command[command.index("--output") + 1]) + directory.mkdir() + result = { + "status": "fail" if candidate else "pass", + "returncode": 7 if candidate else 0, + } + (directory / "diagnostic-results.json").write_text( + json.dumps({"trials": [result]}) + ) + return subprocess.CompletedProcess(command, 1 if candidate else 0) + + monkeypatch.setattr(diag.subprocess, "run", run) + assert diag.compare(args, manifest(), tmp_path) == 1 + report = json.loads((tmp_path / "comparison.json").read_text()) + assert report["verdict"] == "candidate-failed" + assert len(report["trials"]) == 2 + assert report["trials"][1]["returncode"] == 7 From f5bad35e46f0a4b6264f5cd240ee156d28c1f26e Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Sun, 4 Oct 2026 14:54:02 +0100 Subject: [PATCH 3/4] fix: avoid JAX batched LAPACK pool deadlock in validation --- .github/workflows/workspace-validation.yml | 9 ++++++--- docs/release_validation.md | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workspace-validation.yml b/.github/workflows/workspace-validation.yml index 5595312..4408e1d 100644 --- a/.github/workflows/workspace-validation.yml +++ b/.github/workflows/workspace-validation.yml @@ -277,7 +277,8 @@ jobs: # installing the published autolens[optional]; the source checkouts # below shadow the PyAuto packages via PYTHONPATH. pip install "autolens[optional]" - pip install "jax>=0.7,<0.11" "jaxlib>=0.7,<0.11" + # 0.10.2 can deadlock in batched LAPACK; see docs/release_validation.md. + pip install "jax>=0.11.2,<0.12" "jaxlib>=0.11.2,<0.12" # nufftax backs the interferometer TransformerNUFFT and is NOT pulled # transitively by autolens[optional]; without it every interferometer # simulator (and its downstream scripts) fails fast. Matches the pin @@ -349,7 +350,8 @@ jobs: pip install numba # Belt-and-braces JAX install — the [optional] -> [jax] chain # occasionally silently drops jax from test.pypi resolution. - pip install "jax>=0.7,<0.11" "jaxlib>=0.7,<0.11" + # 0.10.2 can deadlock in batched LAPACK; see docs/release_validation.md. + pip install "jax>=0.11.2,<0.12" "jaxlib>=0.11.2,<0.12" pip install "nufftax>=0.6.1,<0.7.0" - name: "Run Python scripts [mode=smoke, source-shadowed, smoke profile]" if: needs.find_scripts.outputs.mode == 'smoke' @@ -561,7 +563,8 @@ jobs: if: steps.gate.outputs.run == 'true' run: | pip install "autolens[optional]" - pip install "jax>=0.7,<0.11" "jaxlib>=0.7,<0.11" + # 0.10.2 can deadlock in batched LAPACK; see docs/release_validation.md. + pip install "jax>=0.11.2,<0.12" "jaxlib>=0.11.2,<0.12" pip install "nufftax>=0.6.1,<0.7.0" pip install jupyter ipynb-py-convert - name: Run notebooks diff --git a/docs/release_validation.md b/docs/release_validation.md index 439d87c..c9968df 100644 --- a/docs/release_validation.md +++ b/docs/release_validation.md @@ -232,3 +232,7 @@ script, or identify a causal repair without reproducible or native evidence. The first exact hosted replay, [37205459198](https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37205459198), passed once in 9.905s and then stalled after 2.6s compilation. Both native tools captured all four Eigen workers waiting in `BlockingCounter::Wait` through `ParallelBatchMap`, `CholeskyFactorization` and `lapack_dpotrf_ffi`. The committed [native witness](../diagnostics/release-37199991757-native.txt) includes the raw artifact's SHA256. There are no FFT/ducc0 frames. [JAX 0.10.2](https://github.com/jax-ml/jax/blob/jax-v0.10.2/jaxlib/cpu/lapack_kernels.cc) schedules LAPACK batch chunks to the thread pool and blocks waiting for them; [JAX 0.11.2](https://github.com/jax-ml/jax/blob/jax-v0.11.2/jaxlib/cpu/lapack_kernels.cc) compiles this parallel path out of open-source builds. This is the source-grounded candidate tested by the comparison, distinct from the historical FFT workaround. No upstream report was filed. + +The interleaved comparison [37206724174](https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37206724174) passed all three controls (15.724/12.671/12.733s) and all three candidates (12.359/12.668/12.167s). It correctly returned **inconclusive** because the control did not stall on that runner. This does not establish a measured failure-rate improvement. The earlier exact replay supplies the native-confirmed failure; the source change removes that captured blocking path, and the candidate executions verify this script with the same release wheels. + +The smoke, integration and notebook dependency recipes now require matching JAX/JAXlib `>=0.11.2,<0.12`, preventing the old `<0.11` override from downgrading into the captured LAPACK deadlock. The original 115-package diagnostic control remains pinned to 0.10.2. No timeout, test selection or release-readiness rule changes. Full release integration has not been repeated: the failed validation remains authoritative until a separately authorized post-merge validation succeeds. From 8721c2e3e747561d4cf010fb2a1a6e431224cbb4 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Sun, 4 Oct 2026 15:43:49 +0100 Subject: [PATCH 4/4] test: validate older and current JAX compatibility endpoints --- .github/scripts/jax_compatibility.py | 118 ++++ .github/workflows/jax-compatibility.yml | 79 +++ .github/workflows/release-diagnostic.yml | 5 - diagnostics/jax-compatibility-20261004.json | 739 ++++++++++++++++++++ docs/release_validation.md | 57 +- 5 files changed, 991 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/jax_compatibility.py create mode 100644 .github/workflows/jax-compatibility.yml create mode 100644 diagnostics/jax-compatibility-20261004.json diff --git a/.github/scripts/jax_compatibility.py b/.github/scripts/jax_compatibility.py new file mode 100644 index 0000000..50f737b --- /dev/null +++ b/.github/scripts/jax_compatibility.py @@ -0,0 +1,118 @@ +"""Bounded numerical/dependency witness for supported JAX versions, not a benchmark.""" +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import os +from pathlib import Path +import platform +import time + + +def run(expected_version, backend): + import numpy as np + import jax + import jax.numpy as jnp + jax.config.update('jax_enable_x64', True) + import blackjax + import optax + import nufftax + + assert jax.__version__ == expected_version + assert importlib.metadata.version('jaxlib') == expected_version + devices = jax.devices() + expected_platform = 'gpu' if backend == 'cuda' else 'cpu' + assert all(d.platform == expected_platform for d in devices), devices + rng = np.random.default_rng(810) + design = rng.normal(size=(40, 12)) + data = rng.normal(size=40) + penalty = np.diag(np.linspace(0.5, 2, 12)) + gram, rhs = design.T @ design, design.T @ data + + def likelihood(theta): + matrix = jnp.asarray(gram) + jnp.exp(theta) * jnp.asarray(penalty) + chol = jnp.linalg.cholesky(matrix) + solution = jax.scipy.linalg.cho_solve((chol, True), jnp.asarray(rhs)) + return -0.5 * (jnp.asarray(data @ data) - rhs @ solution + 2 * jnp.log(jnp.diag(chol)).sum()) + + def reference(theta): + matrix = gram + np.exp(theta) * penalty + return -0.5 * (data @ data - rhs @ np.linalg.solve(matrix, rhs) + np.linalg.slogdet(matrix)[1]) + + theta = np.array([-1.0, 0.1, 1.2]) + compiled = jax.jit(jax.vmap(jax.value_and_grad(likelihood))) + start = time.monotonic() + values, gradients = jax.block_until_ready(compiled(jnp.asarray(theta))) + compile_execute_s = time.monotonic() - start + start = time.monotonic() + for _ in range(10): + jax.block_until_ready(compiled(jnp.asarray(theta))) + steady_s = (time.monotonic() - start) / 10 + reference_values = np.array([reference(x) for x in theta]) + reference_grad = np.array([(reference(x + 1e-5) - reference(x - 1e-5)) / 2e-5 for x in theta]) + np.testing.assert_allclose(values, reference_values, rtol=1e-10, atol=1e-10) + np.testing.assert_allclose(gradients, reference_grad, rtol=1e-6, atol=1e-7) + + # Independent direct DFT oracle for the type-2 NUFFT used by interferometry. + x, y = rng.uniform(-np.pi, np.pi, size=(2, 7)) + modes = rng.normal(size=(8, 8)) + 1j * rng.normal(size=(8, 8)) + frequencies = np.arange(-4, 4) + dft = np.einsum('kj,mj,mk->m', modes, np.exp(-1j * x[:, None] * frequencies), np.exp(-1j * y[:, None] * frequencies)) + nufft = nufftax.nufft2d2(jnp.asarray(x), jnp.asarray(y), jnp.asarray(modes), eps=1e-10) + np.testing.assert_allclose(nufft, dft, rtol=1e-7, atol=1e-7) + + # A real optimizer and a short NUTS chain catch dependency API breakage. + # This checks execution/finite outputs, not posterior convergence. + target = jnp.array([1.0, -2.0]) + objective = lambda p: jnp.sum((p - target) ** 2) + optimizer = optax.adam(0.1) + position = jnp.zeros(2) + state = optimizer.init(position) + @jax.jit + def step(position, state): + updates, state = optimizer.update(jax.grad(objective)(position), state, position) + return optax.apply_updates(position, updates), state + for _ in range(250): + position, state = step(position, state) + np.testing.assert_allclose(position, target, atol=1e-4, rtol=0) + sampler = blackjax.nuts(lambda p: -0.5 * jnp.sum(p ** 2), step_size=0.2, inverse_mass_matrix=jnp.ones(2)) + state = sampler.init(jnp.zeros(2)) + sampler_step = jax.jit(sampler.step) + for key in jax.random.split(jax.random.key(24), 16): + state, info = sampler_step(key, state) + assert np.all(np.isfinite(state.position)) + assert not bool(info.is_divergent) + + return { + 'diagnostic_only': True, 'python': platform.python_version(), + 'backend': backend, 'devices': [str(d) for d in devices], + 'packages': {p: importlib.metadata.version(p) for p in ('jax', 'jaxlib', 'numpy', 'scipy', 'blackjax', 'optax', 'nufftax')}, + 'xla_flags': os.environ.get('XLA_FLAGS'), + 'values': np.asarray(values).tolist(), 'gradients': np.asarray(gradients).tolist(), + 'max_value_error': float(np.max(np.abs(values - reference_values))), + 'max_gradient_error': float(np.max(np.abs(gradients - reference_grad))), + 'max_nufft_error': float(np.max(np.abs(nufft - dft))), + 'compile_execute_s': compile_execute_s, 'steady_s': steady_s, + 'optimizer_position': np.asarray(position).tolist(), 'nuts_steps': 16, + 'status': 'pass', + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--version', required=True) + parser.add_argument('--backend', choices=['cpu', 'cuda'], required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + # Set before JAX import. A missing GPU must fail instead of silently falling back. + os.environ['JAX_PLATFORMS'] = args.backend + os.environ['JAX_PLATFORM_NAME'] = 'gpu' if args.backend == 'cuda' else 'cpu' + os.environ.setdefault('XLA_PYTHON_CLIENT_PREALLOCATE', 'false') + result = run(args.version, args.backend) + args.output.write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps(result), flush=True) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/jax-compatibility.yml b/.github/workflows/jax-compatibility.yml new file mode 100644 index 0000000..42ce306 --- /dev/null +++ b/.github/workflows/jax-compatibility.yml @@ -0,0 +1,79 @@ +name: JAX compatibility endpoints (not release validation) +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/jax-compatibility.yml + - .github/scripts/jax_compatibility.py + - .github/scripts/release_diagnostic.py + - diagnostics/release-37199991757.json +permissions: + contents: read +jobs: + compatibility: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + jax: ['0.9.2', '0.11.2'] + env: + PYTHONPATH: '' + JAX_PLATFORMS: cpu + JAX_PLATFORM_NAME: cpu + XLA_FLAGS: --xla_cpu_multi_thread_eigen=false + steps: + - uses: actions/checkout@v4 + - name: Prepare endpoint pins from preserved incident manifest + id: manifest + env: + ENDPOINT: ${{ matrix.jax }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + data = json.loads(Path('diagnostics/release-37199991757.json').read_text()) + data['packages'].update(jax=os.environ['ENDPOINT'], jaxlib=os.environ['ENDPOINT']) + Path('endpoint.json').write_text(json.dumps(data)) + PY + python .github/scripts/release_diagnostic.py prepare --manifest endpoint.json --output evidence --github-output "$GITHUB_OUTPUT" + - uses: actions/checkout@v4 + with: + repository: ${{ steps.manifest.outputs.workspace_repo }} + ref: ${{ steps.manifest.outputs.workspace_sha }} + path: workspace + persist-credentials: false + - uses: actions/checkout@v4 + with: + repository: PyAutoLabs/PyAutoHands + ref: ${{ steps.manifest.outputs.hands_sha }} + path: hands + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ steps.manifest.outputs.python }} + - name: Install isolated endpoint with unchanged non-JAX pins + run: | + python -m venv endpoint + endpoint/bin/python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --report evidence/install.json -r evidence/requirements.txt + endpoint/bin/python -m pip check + - name: Original likelihood (three bounded fresh processes) + run: >- + endpoint/bin/python .github/scripts/release_diagnostic.py run + --manifest endpoint.json --output evidence --workspace workspace --hands hands --repeats 3 + - name: Numerical and sampler dependency witness + env: + ENDPOINT: ${{ matrix.jax }} + run: >- + timeout --kill-after=5s 180 endpoint/bin/python .github/scripts/jax_compatibility.py + --version "$ENDPOINT" --backend cpu --output evidence/numerics.json + - name: Historical FFT witness with retained workaround + run: >- + timeout --kill-after=5s 120 endpoint/bin/python workspace/.github/scripts/xla_fft_pool_reentrancy_repro.py + > evidence/fft.log 2>&1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: jax-compatibility-${{ matrix.jax }} + path: evidence/ + if-no-files-found: error diff --git a/.github/workflows/release-diagnostic.yml b/.github/workflows/release-diagnostic.yml index 71132f5..7902fbf 100644 --- a/.github/workflows/release-diagnostic.yml +++ b/.github/workflows/release-diagnostic.yml @@ -1,11 +1,6 @@ name: Release Diagnostic (not validation) on: - pull_request: - paths: - - .github/workflows/release-diagnostic.yml - - .github/scripts/release_diagnostic.py - - diagnostics/release-37199991757.json workflow_dispatch: inputs: repeats: diff --git a/diagnostics/jax-compatibility-20261004.json b/diagnostics/jax-compatibility-20261004.json new file mode 100644 index 0000000..0b888e5 --- /dev/null +++ b/diagnostics/jax-compatibility-20261004.json @@ -0,0 +1,739 @@ +{ + "diagnostic_only": true, + "python": "3.12.10", + "gpu": "NVIDIA GeForce RTX 2060 with Max-Q Design; driver 580.97; CUDA12 plugins matched to each JAX version; cuDNN9.10.2.21", + "policy": ">=0.7,<0.12,!=0.10.*,!=0.11.0", + "note": "Preserved range, not blanket certification. Local timings were collected on a contended workstation and are not performance benchmarks. Initial CUDA discovery and DFT-oracle setup failures corrected; full logs retained in Mind scratch. Disposable CUDA installs removed after completed GPU witnesses.", + "numerics": { + "kernels-cpu-0.9.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cpu", + "devices": [ + "TFRT_CPU_0" + ], + "packages": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.24953833472108977, + -0.6743439385131532, + -1.5812558302477793 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.816227334141331e-10, + "max_nufft_error": 8.208770510358971e-11, + "compile_execute_s": 1.0596958999994968, + "steady_s": 0.0008866200001648395, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cpu-0.11.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cpu", + "devices": [ + "cpu:0" + ], + "packages": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.24953833472108977, + -0.6743439385131532, + -1.5812558302477793 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.816227334141331e-10, + "max_nufft_error": 8.208770510358971e-11, + "compile_execute_s": 1.3830478000018047, + "steady_s": 0.002818260000276496, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cuda-0.9.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cuda", + "devices": [ + "cuda:0" + ], + "packages": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.2495383347210898, + -0.6743439385131532, + -1.5812558302477797 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.8162228932492326e-10, + "max_nufft_error": 8.209055434966626e-11, + "compile_execute_s": 1.3817770999994536, + "steady_s": 0.003663880000021891, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cuda-0.11.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cuda", + "devices": [ + "cuda:0" + ], + "packages": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.2495383347210898, + -0.6743439385131532, + -1.5812558302477797 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.8162228932492326e-10, + "max_nufft_error": 8.209055434966626e-11, + "compile_execute_s": 1.6192122000029485, + "steady_s": 0.007460109999738051, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cpu-0.9.2-floors": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cpu", + "devices": [ + "TFRT_CPU_0" + ], + "packages": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "numpy": "2.0.0", + "scipy": "1.13.0", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.24953833472108977, + -0.6743439385131532, + -1.5812558302477793 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.816227334141331e-10, + "max_nufft_error": 8.208770510358971e-11, + "compile_execute_s": 1.1693172000013874, + "steady_s": 0.0003894099998433376, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + } + }, + "workspace_cpu": [ + { + "status": "pass", + "returncode": 0, + "elapsed_s": 55.055, + "cap_s": 300, + "log": "1.log", + "version": "0.9.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 57.419, + "cap_s": 300, + "log": "2.log", + "version": "0.9.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 59.396, + "cap_s": 300, + "log": "3.log", + "version": "0.9.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 184.672, + "cap_s": 300, + "log": "4.log", + "version": "0.9.2", + "backend": "cpu", + "case": "imaging/jax_grad/lp.py", + "script_sha256": "cefb45f3bbf9447a4677ec358aaa37e21ad3a91cd48e15a3adab5973ca6a0678" + }, + { + "status": "timeout", + "returncode": -9, + "elapsed_s": 300.417, + "cap_s": 300, + "log": "5.log", + "version": "0.9.2", + "backend": "cpu", + "case": "point_source/jax_grad/gradient.py", + "script_sha256": "5b56d142083085a198f0b612a9c715e8c9d5c00346934669d2d1c6df195632a3" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 62.969, + "cap_s": 300, + "log": "1.log", + "version": "0.11.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 64.941, + "cap_s": 300, + "log": "2.log", + "version": "0.11.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 70.646, + "cap_s": 300, + "log": "3.log", + "version": "0.11.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 211.715, + "cap_s": 300, + "log": "4.log", + "version": "0.11.2", + "backend": "cpu", + "case": "imaging/jax_grad/lp.py", + "script_sha256": "cefb45f3bbf9447a4677ec358aaa37e21ad3a91cd48e15a3adab5973ca6a0678" + }, + { + "status": "timeout", + "returncode": -9, + "elapsed_s": 300.315, + "cap_s": 300, + "log": "5.log", + "version": "0.11.2", + "backend": "cpu", + "case": "point_source/jax_grad/gradient.py", + "script_sha256": "5b56d142083085a198f0b612a9c715e8c9d5c00346934669d2d1c6df195632a3" + } + ], + "workspace_gpu": [ + { + "status": "pass", + "returncode": 0, + "elapsed_s": 115.299, + "cap_s": 300, + "log": "gpu-workspace-0.9.2.log", + "version": "0.9.2" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 109.857, + "cap_s": 300, + "log": "gpu-workspace-0.11.2.log", + "version": "0.11.2" + } + ], + "fft": [ + { + "version": "0.9.2", + "returncode": 0, + "iterations": 20, + "workaround": "--xla_cpu_multi_thread_eigen=false" + }, + { + "version": "0.11.2", + "returncode": 0, + "iterations": 20, + "workaround": "--xla_cpu_multi_thread_eigen=false" + } + ], + "resolver": [ + { + "package": "autofit", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autofit": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autofit": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autoarray", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autoarray": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autoarray": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autogalaxy", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autogalaxy": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autogalaxy": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autolens", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autolens": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autolens": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autocti", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autocti": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autocti": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + } + ], + "artifacts": { + "compat-results-cpu-0.11.2/1.log": "506582e88b4fd9e3c1a209f72356ac43f8bdc7ea79e98e8a08ed70f76ec4c3aa", + "compat-results-cpu-0.11.2/2.log": "34a7b25139dcb5a0a5b0f18f3fb7541bab236aa08bccb48d15d09d91aa7de192", + "compat-results-cpu-0.11.2/3.log": "f7e3d8f9d126d4fba4092490d83275901460707b8f6b6bea6dbf31ff37d19067", + "compat-results-cpu-0.11.2/4.log": "6b18e86079d09553d3f738b05dbd43e8b2fb2bce0c71f61c28dc2c8c23f93547", + "compat-results-cpu-0.11.2/4.native.txt": "491f772a829081eb99044315971a83009c0bbca4024a1379ef92f26ca0b4d896", + "compat-results-cpu-0.11.2/5.log": "cefb77df603453694f94428c2364aea36f9005b98a9d5fdbab84b4e6edbe454a", + "compat-results-cpu-0.11.2/5.native.txt": "224613ce042e01770207348c41ec7af1bf502d6375192c6b2ebfc5e15bdf564a", + "compat-results-cpu-0.9.2/1.log": "a0ed543f06276d8e9ff22d7949cd9f81763553dd63f1d8f0ea2b29f876dd0696", + "compat-results-cpu-0.9.2/2.log": "377f447838addd6deb60ee68082b8ef3c13d86787d4bff72712668ef40ec3f1a", + "compat-results-cpu-0.9.2/3.log": "f7440970185c1bdf94b55ef03ee5870c1fa1ae175cf1f0ea95bd3dbbfccedc1f", + "compat-results-cpu-0.9.2/4.log": "7c367bc27a879fe7319bc47bcd1c8ac650372b2eee7e0ac352401a64d0fb7499", + "compat-results-cpu-0.9.2/4.native.txt": "3593b7ff238985d5b1b9e3a7e6ad08f17be29a4e7e18e4c8850789405a1a8f40", + "compat-results-cpu-0.9.2/5.log": "98786f4a58cc14a0d2fd382b04d77be2fa93408bd9558557a7d35d2f30dd012a", + "compat-results-cpu-0.9.2/5.native.txt": "3ff1adeb6056b0b67b3212eeb90c388414ad32831a2e8c53497fa2dd141c366e" + }, + "fresh_resolver": { + "status": "pass", + "ignore_installed": true, + "prereleases_enabled": false, + "synthetic_test_version": "2026.10.4.2", + "note": "Synthetic local wheels only; no release/version choice or publication. Ordinary PyPI dependency resolution, no incident constraints. Earlier harness/index/disk failures retained in scratch.", + "selected": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "autolens": "2026.10.4.2", + "nautilus-sampler": "1.0.5", + "autogalaxy": "2026.10.4.2", + "autonerves": "2026.10.4.2", + "jaxnnls": "1.0.1", + "ml_dtypes": "0.6.0", + "numpy": "2.5.3", + "opt_einsum": "3.4.0", + "astropy": "8.0.1", + "autoarray": "2026.10.4.2", + "scipy": "1.17.1", + "autofit": "2026.10.4.2", + "astunparse": "1.6.3", + "corner": "2.2.2", + "dynesty": "2.1.5", + "gprof2dot": "2021.2.21", + "networkx": "3.1", + "psutil": "6.1.0", + "pyvis": "0.3.2", + "timeout-decorator": "0.5.0", + "jax_zero_contour": "2.0.0", + "PyYAML": "6.0.3", + "scikit-learn": "1.8.0", + "threadpoolctl": "3.7.0", + "typing-inspect": "0.9.0", + "anesthetic": "2.16.0", + "astropy-iers-data": "0.2026.9.28.0.59.37", + "decorator": "5.3.1", + "dill": "0.4.1", + "emcee": "3.1.6", + "h5py": "3.16.0", + "joblib": "1.6.0", + "matplotlib": "3.11.2", + "mypy_extensions": "1.1.0", + "numpydoc": "1.11.0", + "optax": "0.2.8", + "packaging": "26.3", + "pyerfa": "2.0.1.5", + "scikit-image": "0.26.0", + "SQLAlchemy": "2.0.54", + "typing_extensions": "4.16.0", + "xxhash": "3.4.1", + "array-api-compat": "1.15.0", + "tqdm": "4.70.1", + "absl-py": "2.5.0", + "cloudpickle": "3.1.2", + "contourpy": "1.4.0", + "cycler": "0.12.1", + "fonttools": "4.66.1", + "greenlet": "3.5.6", + "ImageIO": "2.38.0", + "ipython": "9.12.0", + "Jinja2": "3.1.6", + "jsonpickle": "4.1.3", + "kiwisolver": "1.5.1", + "lazy-loader": "0.6", + "pandas": "3.0.6", + "pillow": "12.3.0", + "pyparsing": "3.3.3", + "python-dateutil": "2.9.0.post0", + "six": "1.17.0", + "Sphinx": "9.1.0", + "tifffile": "2026.9.20", + "wheel": "0.48.0", + "alabaster": "1.0.0", + "babel": "2.18.0", + "docutils": "0.22.4", + "imagesize": "2.0.1", + "ipython_pygments_lexers": "1.1.1", + "jedi": "0.20.0", + "MarkupSafe": "3.0.4", + "matplotlib-inline": "0.2.2", + "pexpect": "4.9.0", + "prompt_toolkit": "3.0.53", + "Pygments": "2.21.0", + "requests": "2.34.2", + "roman-numerals": "4.1.0", + "snowballstemmer": "3.1.1", + "sphinxcontrib-applehelp": "2.0.0", + "sphinxcontrib-devhelp": "2.0.0", + "sphinxcontrib-htmlhelp": "2.1.0", + "sphinxcontrib-jsmath": "1.0.1", + "sphinxcontrib-qthelp": "2.0.0", + "sphinxcontrib-serializinghtml": "2.0.0", + "stack-data": "0.6.3", + "traitlets": "5.16.1", + "asttokens": "3.0.2", + "certifi": "2026.7.22", + "charset-normalizer": "3.5.2", + "executing": "2.2.1", + "idna": "3.20", + "parso": "0.8.7", + "ptyprocess": "0.7.0", + "urllib3": "2.8.0", + "wcwidth": "0.9.1", + "pure_eval": "0.2.4" + }, + "report_sha256": "0a909acaba32ac245d20b1aeeb28bec73ef0637923bfd384e7fc1e5307ecbbbc" + } +} diff --git a/docs/release_validation.md b/docs/release_validation.md index c9968df..cd8d1ac 100644 --- a/docs/release_validation.md +++ b/docs/release_validation.md @@ -198,8 +198,7 @@ is what supplies the `integrate` stage that flips this to GREEN-eligible. # Bounded investigation of a failed wheel run `release-diagnostic.yml` investigates the saved `rectangular_rtu.py` timeout -from run `37199991757`. It runs automatically only on PRs changing that -diagnostic's workflow, runner or manifest, and can also be dispatched manually +from run `37199991757`. It is dispatched manually with `repeats` of 2, 4 or 6 total trials. It does not run the integration matrix. The manifest `diagnostics/release-37199991757.json` preserves the final 115 @@ -236,3 +235,57 @@ The first exact hosted replay, [37205459198](https://github.com/PyAutoLabs/PyAut The interleaved comparison [37206724174](https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37206724174) passed all three controls (15.724/12.671/12.733s) and all three candidates (12.359/12.668/12.167s). It correctly returned **inconclusive** because the control did not stall on that runner. This does not establish a measured failure-rate improvement. The earlier exact replay supplies the native-confirmed failure; the source change removes that captured blocking path, and the candidate executions verify this script with the same release wheels. The smoke, integration and notebook dependency recipes now require matching JAX/JAXlib `>=0.11.2,<0.12`, preventing the old `<0.11` override from downgrading into the captured LAPACK deadlock. The original 115-package diagnostic control remains pinned to 0.10.2. No timeout, test selection or release-readiness rule changes. Full release integration has not been repeated: the failed validation remains authoritative until a separately authorized post-merge validation succeeds. + +### User-facing compatibility policy (2026-10-04) + +The package repair preserves `>=0.7,<0.12` while excluding `0.10.*` and `0.11.0`. +The LAPACK parallel batch path first appears in tagged 0.10.0 source and is disabled +in open-source builds from 0.11.1; 0.9.2 lacks the path. Only 0.10.2 was observed +deadlocking here. Exclusions of the other releases are source-based precautions. +The two diagnostic endpoints are 0.9.2 and 0.11.2, not a new blanket minimum of +0.11.2 or certification of every permitted older version. + +Nerves owns the exclusions and retains its Intel-macOS marker. Each repaired +Fit/Array/Galaxy/Lens/CTI package directly requires `autonerves>2026.10.4.1` to +prevent resolver fallback to an older, permissive Nerves. Hands' three explicit +JAX installs share the exclusions. Heart's integration baseline remains the +0.11.2 series; `jax-compatibility.yml` exercises both endpoints against the saved +wheel stack and original likelihood, plus numerical/sampler and historical FFT +witnesses. The original incident comparison is now manual-only to avoid an +unnecessary old-runtime replay on every PR update. + +**Release ordering:** merge/publish the policy-bearing Nerves before the repaired +family wheels. The strict bound rejects the old release's `.devN` and `.postN` +variants too; a same-day rehearsal must use a higher base version chosen by the +human release process. Source CI builds use the existing `setup.py` development +version `9999.0.0.dev0` and satisfy the guard. None of this retroactively repairs +published wheels or prevents an unconstrained installer from selecting an entirely +older family. Existing lockfiles require an explicit repaired-family upgrade. + +[Local receipts](../diagnostics/jax-compatibility-20261004.json) record: + +- Original rectangular likelihood, including its existing JIT/vmap assertions: + three CPU passes and one actual CUDA pass per version, with the same incident + wheels, 113 non-JAX pins and saved dataset. Local Python is 3.12.10, not the + incident's 3.12.14; the hosted workflow pins 3.12.14. +- Imaging light-profile gradient checks pass for both versions against finite + differences. Independent regularized-likelihood/Cholesky, gradient, NUFFT/direct + DFT, Optax optimizer and short BlackJAX NUTS execution checks pass on CPU and GPU. + The older endpoint also passes with NumPy2.0.0/SciPy1.13.0. The short NUTS run + tests finite execution, not posterior convergence. +- The historical FFT reproducer completes 20 iterations on both endpoints with + `--xla_cpu_multi_thread_eigen=false`. The separate FFT workaround is retained. +- All six real built wheels have the intended metadata. 25 live resolver cases + across the five consumer packages accept both endpoints and reject 0.10.2, + 0.11.0 and old Nerves, with older published candidates still available. A fresh `--ignore-installed` + resolver run without prereleases also selects the protected family and JAX0.9.2 + using synthetic stable local wheels (no release or version selection). + +**Incomplete broader check:** `point_source/jax_grad/gradient.py` completed its +solved-source finite-difference checks on both versions, then exceeded the 300s +local diagnostic cap in the later portion on both. Logs/native-capture attempts +and hashes are retained. This does not establish a version-specific regression +or its cause, and is not a passing result. No cap or test selection was changed +in production. Local timings were collected with other validation running and +must not be interpreted as a performance comparison. Full release validation +remains failed and must be assessed separately after human merge/authorization.