diff --git a/.github/scripts/jax_compatibility.py b/.github/scripts/jax_compatibility.py new file mode 100644 index 0000000..50f737b --- /dev/null +++ b/.github/scripts/jax_compatibility.py @@ -0,0 +1,118 @@ +"""Bounded numerical/dependency witness for supported JAX versions, not a benchmark.""" +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import os +from pathlib import Path +import platform +import time + + +def run(expected_version, backend): + import numpy as np + import jax + import jax.numpy as jnp + jax.config.update('jax_enable_x64', True) + import blackjax + import optax + import nufftax + + assert jax.__version__ == expected_version + assert importlib.metadata.version('jaxlib') == expected_version + devices = jax.devices() + expected_platform = 'gpu' if backend == 'cuda' else 'cpu' + assert all(d.platform == expected_platform for d in devices), devices + rng = np.random.default_rng(810) + design = rng.normal(size=(40, 12)) + data = rng.normal(size=40) + penalty = np.diag(np.linspace(0.5, 2, 12)) + gram, rhs = design.T @ design, design.T @ data + + def likelihood(theta): + matrix = jnp.asarray(gram) + jnp.exp(theta) * jnp.asarray(penalty) + chol = jnp.linalg.cholesky(matrix) + solution = jax.scipy.linalg.cho_solve((chol, True), jnp.asarray(rhs)) + return -0.5 * (jnp.asarray(data @ data) - rhs @ solution + 2 * jnp.log(jnp.diag(chol)).sum()) + + def reference(theta): + matrix = gram + np.exp(theta) * penalty + return -0.5 * (data @ data - rhs @ np.linalg.solve(matrix, rhs) + np.linalg.slogdet(matrix)[1]) + + theta = np.array([-1.0, 0.1, 1.2]) + compiled = jax.jit(jax.vmap(jax.value_and_grad(likelihood))) + start = time.monotonic() + values, gradients = jax.block_until_ready(compiled(jnp.asarray(theta))) + compile_execute_s = time.monotonic() - start + start = time.monotonic() + for _ in range(10): + jax.block_until_ready(compiled(jnp.asarray(theta))) + steady_s = (time.monotonic() - start) / 10 + reference_values = np.array([reference(x) for x in theta]) + reference_grad = np.array([(reference(x + 1e-5) - reference(x - 1e-5)) / 2e-5 for x in theta]) + np.testing.assert_allclose(values, reference_values, rtol=1e-10, atol=1e-10) + np.testing.assert_allclose(gradients, reference_grad, rtol=1e-6, atol=1e-7) + + # Independent direct DFT oracle for the type-2 NUFFT used by interferometry. + x, y = rng.uniform(-np.pi, np.pi, size=(2, 7)) + modes = rng.normal(size=(8, 8)) + 1j * rng.normal(size=(8, 8)) + frequencies = np.arange(-4, 4) + dft = np.einsum('kj,mj,mk->m', modes, np.exp(-1j * x[:, None] * frequencies), np.exp(-1j * y[:, None] * frequencies)) + nufft = nufftax.nufft2d2(jnp.asarray(x), jnp.asarray(y), jnp.asarray(modes), eps=1e-10) + np.testing.assert_allclose(nufft, dft, rtol=1e-7, atol=1e-7) + + # A real optimizer and a short NUTS chain catch dependency API breakage. + # This checks execution/finite outputs, not posterior convergence. + target = jnp.array([1.0, -2.0]) + objective = lambda p: jnp.sum((p - target) ** 2) + optimizer = optax.adam(0.1) + position = jnp.zeros(2) + state = optimizer.init(position) + @jax.jit + def step(position, state): + updates, state = optimizer.update(jax.grad(objective)(position), state, position) + return optax.apply_updates(position, updates), state + for _ in range(250): + position, state = step(position, state) + np.testing.assert_allclose(position, target, atol=1e-4, rtol=0) + sampler = blackjax.nuts(lambda p: -0.5 * jnp.sum(p ** 2), step_size=0.2, inverse_mass_matrix=jnp.ones(2)) + state = sampler.init(jnp.zeros(2)) + sampler_step = jax.jit(sampler.step) + for key in jax.random.split(jax.random.key(24), 16): + state, info = sampler_step(key, state) + assert np.all(np.isfinite(state.position)) + assert not bool(info.is_divergent) + + return { + 'diagnostic_only': True, 'python': platform.python_version(), + 'backend': backend, 'devices': [str(d) for d in devices], + 'packages': {p: importlib.metadata.version(p) for p in ('jax', 'jaxlib', 'numpy', 'scipy', 'blackjax', 'optax', 'nufftax')}, + 'xla_flags': os.environ.get('XLA_FLAGS'), + 'values': np.asarray(values).tolist(), 'gradients': np.asarray(gradients).tolist(), + 'max_value_error': float(np.max(np.abs(values - reference_values))), + 'max_gradient_error': float(np.max(np.abs(gradients - reference_grad))), + 'max_nufft_error': float(np.max(np.abs(nufft - dft))), + 'compile_execute_s': compile_execute_s, 'steady_s': steady_s, + 'optimizer_position': np.asarray(position).tolist(), 'nuts_steps': 16, + 'status': 'pass', + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--version', required=True) + parser.add_argument('--backend', choices=['cpu', 'cuda'], required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + # Set before JAX import. A missing GPU must fail instead of silently falling back. + os.environ['JAX_PLATFORMS'] = args.backend + os.environ['JAX_PLATFORM_NAME'] = 'gpu' if args.backend == 'cuda' else 'cpu' + os.environ.setdefault('XLA_PYTHON_CLIENT_PREALLOCATE', 'false') + result = run(args.version, args.backend) + args.output.write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps(result), flush=True) + + +if __name__ == '__main__': + main() diff --git a/.github/scripts/release_diagnostic.py b/.github/scripts/release_diagnostic.py new file mode 100644 index 0000000..bed1bf9 --- /dev/null +++ b/.github/scripts/release_diagnostic.py @@ -0,0 +1,379 @@ +"""Bounded wheel-only investigation; never emits release-validation evidence.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import importlib.util +import json +import os +from pathlib import Path +import platform +import re +import signal +import subprocess +import sys +import time + + +LIBRARIES = ("autonerves", "autofit", "autoarray", "autogalaxy", "autolens") + + +def load_manifest(path): + data = json.loads(Path(path).read_text()) + if data.get("schema_version") != 1: + raise ValueError("unsupported diagnostic manifest") + for key in ("workspace_sha", "hands_sha"): + if not re.fullmatch(r"[0-9a-f]{40}", data.get(key, "")): + raise ValueError(f"{key} must be an immutable commit") + if not re.fullmatch( + r"[A-Za-z0-9_-]+/[A-Za-z0-9_-]+", data.get("workspace_repo", "") + ): + raise ValueError("invalid workspace repository") + if not re.fullmatch(r"3\.\d+\.\d+", data.get("python", "")): + raise ValueError("pin the Python patch version") + script = Path(data.get("script", "")) + if script.is_absolute() or ".." in script.parts or script.suffix != ".py": + raise ValueError("script must be a relative Python path inside scripts/") + packages = data.get("packages", {}) + if not all(name in packages for name in (*LIBRARIES, "jax", "jaxlib")): + raise ValueError("missing library/runtime package pins") + for name, version in packages.items(): + if not re.fullmatch(r"[A-Za-z0-9_.-]+", name) or not re.fullmatch( + r"[0-9][A-Za-z0-9_.+!-]*", version + ): + raise ValueError("package pins must be names and exact versions") + if "candidate_jax" in data and not re.fullmatch( + r"[0-9]+\.[0-9]+\.[0-9]+", data["candidate_jax"] + ): + raise ValueError("candidate JAX must be an exact release version") + return data + + +def prepare(manifest, output, github_output=None): + output.mkdir(parents=True, exist_ok=True) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + (output / "requirements.txt").write_text( + "".join( + f"{name}=={version}\n" + for name, version in sorted(manifest["packages"].items()) + ) + ) + if github_output: + with Path(github_output).open("a") as stream: + for key in ("python", "workspace_repo", "workspace_sha", "hands_sha"): + print(f"{key}={manifest[key]}", file=stream) + + +def provenance(manifest, workspace, hands): + errors = [] + receipt = { + "diagnostic_only": True, + "python": platform.python_version(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "affinity": sorted(os.sched_getaffinity(0)), + "packages": {}, + "imports": {}, + } + receipt["runner_image"] = { + key: os.environ.get(key) for key in ("ImageOS", "ImageVersion") + } + receipt["cpu_quota"] = {} + for path in ("/sys/fs/cgroup/cpu.max", "/sys/fs/cgroup/cpuset.cpus.effective"): + try: + receipt["cpu_quota"][path] = Path(path).read_text().strip() + except OSError: + receipt["cpu_quota"][path] = None + for label, root, expected in ( + ("workspace", workspace, manifest["workspace_sha"]), + ("hands", hands, manifest["hands_sha"]), + ): + actual = subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip() + receipt[label + "_sha"] = actual + if actual != expected: + errors.append(f"{label} commit differs from manifest") + if receipt["python"] != manifest["python"]: + errors.append("Python patch version differs from manifest") + for name, expected in manifest["packages"].items(): + try: + actual = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + actual = None + receipt["packages"][name] = actual + if actual != expected: + errors.append(f"{name}: expected {expected}, installed {actual}") + for name in (*LIBRARIES, "jax", "jaxlib"): + spec = importlib.util.find_spec(name) + origin = spec.origin if spec else None + receipt["imports"][name] = origin + if not origin or "site-packages" not in Path(origin).parts: + errors.append(f"{name} is not imported from installed wheels") + receipt["errors"] = errors + return receipt + + +def native_dump(pid, output): + """Collect while the child lives; each helper has its own short deadline.""" + blocks = [] + for task in sorted(Path(f"/proc/{pid}/task").glob("*")): + try: + blocks.append( + f"{task.name}: {(task / 'comm').read_text().strip()} " + f"{(task / 'wchan').read_text().strip()}" + ) + except OSError as error: + blocks.append(str(error)) + commands = [ + ["py-spy", "dump", "--native", "--pid", str(pid)], + [ + "gdb", + "--batch", + "-p", + str(pid), + "-ex", + "set pagination off", + "-ex", + "thread apply all bt", + "-ex", + "detach", + ], + ] + for command in commands: + blocks.append("$ " + " ".join(command)) + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=20) + blocks.extend([f"exit={result.returncode}", result.stdout, result.stderr]) + except (OSError, subprocess.TimeoutExpired) as error: + blocks.append(str(error)) + if isinstance(error, subprocess.TimeoutExpired): + for partial in (error.stdout, error.stderr): + if partial: + blocks.append( + partial.decode(errors="replace") + if isinstance(partial, bytes) + else partial + ) + output.write_text("\n".join(blocks)) + + +def trial(command, cwd, env, output, *, cap=300, dump_after=120, dump=native_dump): + started = time.monotonic() + timed_out = False + with output.open("w") as log: + proc = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + try: + proc.wait(timeout=dump_after) + except subprocess.TimeoutExpired: + dump(proc.pid, output.with_suffix(".native.txt")) + try: + proc.wait(timeout=max(0, cap - (time.monotonic() - started))) + except subprocess.TimeoutExpired: + timed_out = True + finally: + # Also retire any simulator descendants; none may survive this trial. + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + return { + "status": ( + "timeout" if timed_out else "pass" if proc.returncode == 0 else "fail" + ), + "returncode": proc.returncode, + "elapsed_s": round(time.monotonic() - started, 3), + "cap_s": cap, + "log": output.name, + } + + +def comparison_verdict(rows, expected): + if not rows or expected % 2: + return "incomplete" + order = ["control", "candidate"] * (expected // 2) + if len(rows) > expected or [row["arm"] for row in rows] != order[: len(rows)]: + return "invalid-order" + if any(row["status"] != "pass" for row in rows if row["arm"] == "candidate"): + return "candidate-failed" + if any(row["status"] not in {"pass", "timeout"} for row in rows): + return "control-failed" + if len(rows) != expected: + return "incomplete" + if not any( + row.get("cholesky_pool_stall") for row in rows if row["arm"] == "control" + ): + return "inconclusive" + return "captured-stall-avoided" + + +def compare(args, manifest, output): + """ABAB on one runner and dataset; keep adverse controls verbatim.""" + if not args.candidate_python or args.repeats % 2: + raise ValueError("comparison needs candidate Python and 2, 4 or 6 trials") + rows = [] + report = { + "diagnostic_only": True, + "source_run": manifest["source_run"], + "trials": rows, + } + for index in range(args.repeats): + arm = "control" if index % 2 == 0 else "candidate" + directory = output / f"{index + 1}-{arm}" + # Resolving a venv interpreter symlink would escape into base Python. + python = ( + sys.executable + if arm == "control" + else str(args.candidate_python.absolute()) + ) + if directory.exists(): + raise ValueError( + f"preserve previous evidence; choose a fresh output directory: {directory}" + ) + command = [ + python, + str(Path(__file__).resolve()), + "run", + "--manifest", + str(args.manifest.resolve()), + "--output", + str(directory), + "--workspace", + str(args.workspace.resolve()), + "--hands", + str(args.hands.resolve()), + "--repeats", + "1", + ] + if arm == "candidate": + command.append("--candidate") + completed = subprocess.run(command, timeout=360) + result_path = directory / "diagnostic-results.json" + if not result_path.exists(): + raise RuntimeError( + f"{arm} failed before a measured trial (exit {completed.returncode})" + ) + result = json.loads(result_path.read_text())["trials"][0] + expected_exit = 0 if result["status"] == "pass" else 1 + if completed.returncode != expected_exit: + raise RuntimeError(f"{arm} runner exit disagrees with its trial artifact") + result["arm"] = arm + result["directory"] = directory.name + native_path = directory / "trial-1.native.txt" + native = native_path.read_text() if native_path.exists() else "" + result["cholesky_pool_stall"] = result["status"] == "timeout" and all( + native.count(marker) >= 4 + for marker in ( + "BlockingCounter::Wait", + "ParallelBatchMap", + "CholeskyFactorization", + ) + ) + rows.append(result) + report["verdict"] = comparison_verdict(rows, args.repeats) + (output / "comparison.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(result), flush=True) + if result["status"] not in {"pass", "timeout"} or ( + arm == "candidate" and result["status"] != "pass" + ): + return 1 + print( + "Diagnostic comparison: " + report["verdict"] + "; not release clearance", + flush=True, + ) + return 0 if report["verdict"] == "captured-stall-avoided" else 2 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("prepare", "run", "compare")) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--github-output") + parser.add_argument("--workspace", type=Path) + parser.add_argument("--hands", type=Path) + parser.add_argument("--repeats", type=int, default=6) + parser.add_argument("--candidate", action="store_true") + parser.add_argument("--candidate-python", type=Path) + args = parser.parse_args(argv) + if not 1 <= args.repeats <= 6: + parser.error("repeats must be between 1 and 6") + manifest = load_manifest(args.manifest) + if args.candidate: + manifest["packages"].update( + jax=manifest["candidate_jax"], jaxlib=manifest["candidate_jax"] + ) + output = args.output.resolve() + prepare(manifest, output, args.github_output) + if args.mode == "prepare": + return 0 + if not args.workspace or not args.hands: + parser.error("run needs --workspace and --hands") + if args.mode == "compare": + if args.candidate: + parser.error("comparison must start from the unmodified control manifest") + return compare(args, manifest, output) + workspace, hands = args.workspace.resolve(), args.hands.resolve() + receipt = provenance(manifest, workspace, hands) + (output / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") + if receipt["errors"]: + print("Provenance mismatch: " + "; ".join(receipt["errors"]), flush=True) + return 2 + # Import only the pinned Hands resolver into this parent. Child imports + # never inherit source directories from sys.path. + sys.path.insert(0, str(hands / "autohands")) + from env_config import build_env_for_script, load_env_config + + script = workspace / "scripts" / manifest["script"] + if not script.is_file() or not script.resolve().is_relative_to( + workspace / "scripts" + ): + raise ValueError("script missing or outside workspace") + config = load_env_config(workspace / "config/build/profile_release.yaml") + env = build_env_for_script(script, config) + env.pop("PYTHONPATH", None) + env.update(PYTHONUNBUFFERED="1", JAX_TRACEBACK_FILTERING="off") + receipt["environment"] = { + key: value + for key, value in env.items() + if key.startswith(("PYAUTO_", "JAX_", "XLA_", "OMP_", "MKL_", "OPENBLAS_")) + } + (output / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") + results = { + "diagnostic_only": True, + "source_run": manifest["source_run"], + "trials": [], + } + for index in range(args.repeats): + result = trial( + [sys.executable, str(script)], + workspace, + env, + output / f"trial-{index + 1}.log", + ) + results["trials"].append(result) + (output / "diagnostic-results.json").write_text( + json.dumps(results, indent=2) + "\n" + ) + print(json.dumps(result), flush=True) + if result["status"] != "pass": + return 1 + print( + "Diagnostic repetitions passed; this is not release-validation clearance.", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/jax-compatibility.yml b/.github/workflows/jax-compatibility.yml new file mode 100644 index 0000000..42ce306 --- /dev/null +++ b/.github/workflows/jax-compatibility.yml @@ -0,0 +1,79 @@ +name: JAX compatibility endpoints (not release validation) +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/jax-compatibility.yml + - .github/scripts/jax_compatibility.py + - .github/scripts/release_diagnostic.py + - diagnostics/release-37199991757.json +permissions: + contents: read +jobs: + compatibility: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + jax: ['0.9.2', '0.11.2'] + env: + PYTHONPATH: '' + JAX_PLATFORMS: cpu + JAX_PLATFORM_NAME: cpu + XLA_FLAGS: --xla_cpu_multi_thread_eigen=false + steps: + - uses: actions/checkout@v4 + - name: Prepare endpoint pins from preserved incident manifest + id: manifest + env: + ENDPOINT: ${{ matrix.jax }} + run: | + python - <<'PY' + import json, os + from pathlib import Path + data = json.loads(Path('diagnostics/release-37199991757.json').read_text()) + data['packages'].update(jax=os.environ['ENDPOINT'], jaxlib=os.environ['ENDPOINT']) + Path('endpoint.json').write_text(json.dumps(data)) + PY + python .github/scripts/release_diagnostic.py prepare --manifest endpoint.json --output evidence --github-output "$GITHUB_OUTPUT" + - uses: actions/checkout@v4 + with: + repository: ${{ steps.manifest.outputs.workspace_repo }} + ref: ${{ steps.manifest.outputs.workspace_sha }} + path: workspace + persist-credentials: false + - uses: actions/checkout@v4 + with: + repository: PyAutoLabs/PyAutoHands + ref: ${{ steps.manifest.outputs.hands_sha }} + path: hands + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ steps.manifest.outputs.python }} + - name: Install isolated endpoint with unchanged non-JAX pins + run: | + python -m venv endpoint + endpoint/bin/python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --report evidence/install.json -r evidence/requirements.txt + endpoint/bin/python -m pip check + - name: Original likelihood (three bounded fresh processes) + run: >- + endpoint/bin/python .github/scripts/release_diagnostic.py run + --manifest endpoint.json --output evidence --workspace workspace --hands hands --repeats 3 + - name: Numerical and sampler dependency witness + env: + ENDPOINT: ${{ matrix.jax }} + run: >- + timeout --kill-after=5s 180 endpoint/bin/python .github/scripts/jax_compatibility.py + --version "$ENDPOINT" --backend cpu --output evidence/numerics.json + - name: Historical FFT witness with retained workaround + run: >- + timeout --kill-after=5s 120 endpoint/bin/python workspace/.github/scripts/xla_fft_pool_reentrancy_repro.py + > evidence/fft.log 2>&1 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: jax-compatibility-${{ matrix.jax }} + path: evidence/ + if-no-files-found: error diff --git a/.github/workflows/release-diagnostic.yml b/.github/workflows/release-diagnostic.yml new file mode 100644 index 0000000..7902fbf --- /dev/null +++ b/.github/workflows/release-diagnostic.yml @@ -0,0 +1,83 @@ +name: Release Diagnostic (not validation) + +on: + workflow_dispatch: + inputs: + repeats: + description: "Alternating control/candidate processes (2, 4 or 6)" + type: number + default: 6 + +permissions: + contents: read + +jobs: + diagnose: + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + PYTHONPATH: "" + MANIFEST: diagnostics/release-37199991757.json + steps: + - uses: actions/checkout@v4 + - name: Validate immutable manifest and write requirements + id: manifest + run: >- + python3 .github/scripts/release_diagnostic.py prepare + --manifest "$MANIFEST" --output diagnostic-results + --github-output "$GITHUB_OUTPUT" + - uses: actions/checkout@v4 + with: + repository: ${{ steps.manifest.outputs.workspace_repo }} + ref: ${{ steps.manifest.outputs.workspace_sha }} + path: diagnostic-workspace + persist-credentials: false + - uses: actions/checkout@v4 + with: + repository: PyAutoLabs/PyAutoHands + ref: ${{ steps.manifest.outputs.hands_sha }} + path: diagnostic-hands + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ steps.manifest.outputs.python }} + - name: Install recovered wheel environment and native inspection tools + run: | + python -m venv diagnostic-env + diagnostic-env/bin/python -m pip install \ + --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple \ + --report diagnostic-results/pip-install.json \ + -r diagnostic-results/requirements.txt + diagnostic-env/bin/python -m pip install py-spy + python .github/scripts/release_diagnostic.py prepare --candidate \ + --manifest "$MANIFEST" --output diagnostic-results/candidate-install + python -m venv diagnostic-candidate + diagnostic-candidate/bin/python -m pip install \ + --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple \ + --report diagnostic-results/candidate-install/pip-install.json \ + -r diagnostic-results/candidate-install/requirements.txt + echo "$PWD/diagnostic-env/bin" >> "$GITHUB_PATH" + sudo apt-get update -qq + sudo apt-get install -y -qq gdb + # Ephemeral runner only; allow sibling debuggers to attach to Python. + sudo sysctl -w kernel.yama.ptrace_scope=0 + - name: Interleaved runtime comparison against the same release wheels + env: + REPEATS: ${{ inputs.repeats || 6 }} + run: >- + diagnostic-env/bin/python .github/scripts/release_diagnostic.py compare + --manifest "$MANIFEST" --output diagnostic-results + --workspace diagnostic-workspace --hands diagnostic-hands + --candidate-python diagnostic-candidate/bin/python + --repeats "$REPEATS" + - name: Preserve diagnostic evidence, including failure and native stacks + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-diagnostic-${{ github.run_id }} + path: | + diagnostic-results/ + diagnostic-workspace/dataset/multi_dataset/lens_sersic/*.fits + if-no-files-found: error diff --git a/.github/workflows/workspace-validation.yml b/.github/workflows/workspace-validation.yml index 5595312..4408e1d 100644 --- a/.github/workflows/workspace-validation.yml +++ b/.github/workflows/workspace-validation.yml @@ -277,7 +277,8 @@ jobs: # installing the published autolens[optional]; the source checkouts # below shadow the PyAuto packages via PYTHONPATH. pip install "autolens[optional]" - pip install "jax>=0.7,<0.11" "jaxlib>=0.7,<0.11" + # 0.10.2 can deadlock in batched LAPACK; see docs/release_validation.md. + pip install "jax>=0.11.2,<0.12" "jaxlib>=0.11.2,<0.12" # nufftax backs the interferometer TransformerNUFFT and is NOT pulled # transitively by autolens[optional]; without it every interferometer # simulator (and its downstream scripts) fails fast. Matches the pin @@ -349,7 +350,8 @@ jobs: pip install numba # Belt-and-braces JAX install — the [optional] -> [jax] chain # occasionally silently drops jax from test.pypi resolution. - pip install "jax>=0.7,<0.11" "jaxlib>=0.7,<0.11" + # 0.10.2 can deadlock in batched LAPACK; see docs/release_validation.md. + pip install "jax>=0.11.2,<0.12" "jaxlib>=0.11.2,<0.12" pip install "nufftax>=0.6.1,<0.7.0" - name: "Run Python scripts [mode=smoke, source-shadowed, smoke profile]" if: needs.find_scripts.outputs.mode == 'smoke' @@ -561,7 +563,8 @@ jobs: if: steps.gate.outputs.run == 'true' run: | pip install "autolens[optional]" - pip install "jax>=0.7,<0.11" "jaxlib>=0.7,<0.11" + # 0.10.2 can deadlock in batched LAPACK; see docs/release_validation.md. + pip install "jax>=0.11.2,<0.12" "jaxlib>=0.11.2,<0.12" pip install "nufftax>=0.6.1,<0.7.0" pip install jupyter ipynb-py-convert - name: Run notebooks diff --git a/diagnostics/jax-compatibility-20261004.json b/diagnostics/jax-compatibility-20261004.json new file mode 100644 index 0000000..0b888e5 --- /dev/null +++ b/diagnostics/jax-compatibility-20261004.json @@ -0,0 +1,739 @@ +{ + "diagnostic_only": true, + "python": "3.12.10", + "gpu": "NVIDIA GeForce RTX 2060 with Max-Q Design; driver 580.97; CUDA12 plugins matched to each JAX version; cuDNN9.10.2.21", + "policy": ">=0.7,<0.12,!=0.10.*,!=0.11.0", + "note": "Preserved range, not blanket certification. Local timings were collected on a contended workstation and are not performance benchmarks. Initial CUDA discovery and DFT-oracle setup failures corrected; full logs retained in Mind scratch. Disposable CUDA installs removed after completed GPU witnesses.", + "numerics": { + "kernels-cpu-0.9.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cpu", + "devices": [ + "TFRT_CPU_0" + ], + "packages": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.24953833472108977, + -0.6743439385131532, + -1.5812558302477793 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.816227334141331e-10, + "max_nufft_error": 8.208770510358971e-11, + "compile_execute_s": 1.0596958999994968, + "steady_s": 0.0008866200001648395, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cpu-0.11.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cpu", + "devices": [ + "cpu:0" + ], + "packages": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.24953833472108977, + -0.6743439385131532, + -1.5812558302477793 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.816227334141331e-10, + "max_nufft_error": 8.208770510358971e-11, + "compile_execute_s": 1.3830478000018047, + "steady_s": 0.002818260000276496, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cuda-0.9.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cuda", + "devices": [ + "cuda:0" + ], + "packages": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.2495383347210898, + -0.6743439385131532, + -1.5812558302477797 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.8162228932492326e-10, + "max_nufft_error": 8.209055434966626e-11, + "compile_execute_s": 1.3817770999994536, + "steady_s": 0.003663880000021891, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cuda-0.11.2": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cuda", + "devices": [ + "cuda:0" + ], + "packages": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "numpy": "2.5.3", + "scipy": "1.17.1", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.2495383347210898, + -0.6743439385131532, + -1.5812558302477797 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.8162228932492326e-10, + "max_nufft_error": 8.209055434966626e-11, + "compile_execute_s": 1.6192122000029485, + "steady_s": 0.007460109999738051, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + }, + "kernels-cpu-0.9.2-floors": { + "diagnostic_only": true, + "python": "3.12.10", + "backend": "cpu", + "devices": [ + "TFRT_CPU_0" + ], + "packages": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "numpy": "2.0.0", + "scipy": "1.13.0", + "blackjax": "1.7.1", + "optax": "0.2.8", + "nufftax": "0.6.1" + }, + "xla_flags": "--xla_cpu_multi_thread_eigen=false", + "values": [ + -30.602441710810105, + -31.076385661423217, + -32.26494197253405 + ], + "gradients": [ + -0.24953833472108977, + -0.6743439385131532, + -1.5812558302477793 + ], + "max_value_error": 0.0, + "max_gradient_error": 3.816227334141331e-10, + "max_nufft_error": 8.208770510358971e-11, + "compile_execute_s": 1.1693172000013874, + "steady_s": 0.0003894099998433376, + "optimizer_position": [ + 0.9999985821120013, + -2.000000279564477 + ], + "nuts_steps": 16, + "status": "pass" + } + }, + "workspace_cpu": [ + { + "status": "pass", + "returncode": 0, + "elapsed_s": 55.055, + "cap_s": 300, + "log": "1.log", + "version": "0.9.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 57.419, + "cap_s": 300, + "log": "2.log", + "version": "0.9.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 59.396, + "cap_s": 300, + "log": "3.log", + "version": "0.9.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 184.672, + "cap_s": 300, + "log": "4.log", + "version": "0.9.2", + "backend": "cpu", + "case": "imaging/jax_grad/lp.py", + "script_sha256": "cefb45f3bbf9447a4677ec358aaa37e21ad3a91cd48e15a3adab5973ca6a0678" + }, + { + "status": "timeout", + "returncode": -9, + "elapsed_s": 300.417, + "cap_s": 300, + "log": "5.log", + "version": "0.9.2", + "backend": "cpu", + "case": "point_source/jax_grad/gradient.py", + "script_sha256": "5b56d142083085a198f0b612a9c715e8c9d5c00346934669d2d1c6df195632a3" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 62.969, + "cap_s": 300, + "log": "1.log", + "version": "0.11.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 64.941, + "cap_s": 300, + "log": "2.log", + "version": "0.11.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 70.646, + "cap_s": 300, + "log": "3.log", + "version": "0.11.2", + "backend": "cpu", + "case": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "script_sha256": "5cc4e77ecf2c3bddaeccbba7aeb0094ff942eff086af5d35a6eb9c6cf47d48bc" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 211.715, + "cap_s": 300, + "log": "4.log", + "version": "0.11.2", + "backend": "cpu", + "case": "imaging/jax_grad/lp.py", + "script_sha256": "cefb45f3bbf9447a4677ec358aaa37e21ad3a91cd48e15a3adab5973ca6a0678" + }, + { + "status": "timeout", + "returncode": -9, + "elapsed_s": 300.315, + "cap_s": 300, + "log": "5.log", + "version": "0.11.2", + "backend": "cpu", + "case": "point_source/jax_grad/gradient.py", + "script_sha256": "5b56d142083085a198f0b612a9c715e8c9d5c00346934669d2d1c6df195632a3" + } + ], + "workspace_gpu": [ + { + "status": "pass", + "returncode": 0, + "elapsed_s": 115.299, + "cap_s": 300, + "log": "gpu-workspace-0.9.2.log", + "version": "0.9.2" + }, + { + "status": "pass", + "returncode": 0, + "elapsed_s": 109.857, + "cap_s": 300, + "log": "gpu-workspace-0.11.2.log", + "version": "0.11.2" + } + ], + "fft": [ + { + "version": "0.9.2", + "returncode": 0, + "iterations": 20, + "workaround": "--xla_cpu_multi_thread_eigen=false" + }, + { + "version": "0.11.2", + "returncode": 0, + "iterations": 20, + "workaround": "--xla_cpu_multi_thread_eigen=false" + } + ], + "resolver": [ + { + "package": "autofit", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autofit": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autofit": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autofit", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ebe7d0c1a756214258b6c439cf2966293ea7592d9edb33f2caf7dc70e573ca56" + }, + { + "package": "autoarray", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autoarray": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autoarray": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autoarray", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "2f824bdc82fab9d4d31254e20fee860eef5645dad051fc59894dd341909f9465" + }, + { + "package": "autogalaxy", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autogalaxy": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autogalaxy": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autogalaxy", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "33b3a4e566f25f7597c88fe528cae1a1078460496d0bf575d7c112fdce88d3c2" + }, + { + "package": "autolens", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autolens": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autolens": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autolens", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "ef1fbb566e4153db8bd2a54e5185920c84baf73a6b48eeeeda0dc0a908e171bb" + }, + { + "package": "autocti", + "case": "old", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "autocti": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "new", + "expected_resolution": true, + "returncode": 0, + "pass": true, + "selected": { + "jax": "0.11.2", + "jaxlib": "0.11.2", + "autocti": "9999.0.0.dev0", + "autonerves": "9999.0.0.dev0" + }, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "blocked-0102", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "blocked-0110", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + }, + { + "package": "autocti", + "case": "old-nerves", + "expected_resolution": false, + "returncode": 1, + "pass": true, + "selected": {}, + "wheel_sha256": "50c6ac5796889ed5605bfd9e281d28ad73abe8056df95152e0ae1ce32efb6200" + } + ], + "artifacts": { + "compat-results-cpu-0.11.2/1.log": "506582e88b4fd9e3c1a209f72356ac43f8bdc7ea79e98e8a08ed70f76ec4c3aa", + "compat-results-cpu-0.11.2/2.log": "34a7b25139dcb5a0a5b0f18f3fb7541bab236aa08bccb48d15d09d91aa7de192", + "compat-results-cpu-0.11.2/3.log": "f7e3d8f9d126d4fba4092490d83275901460707b8f6b6bea6dbf31ff37d19067", + "compat-results-cpu-0.11.2/4.log": "6b18e86079d09553d3f738b05dbd43e8b2fb2bce0c71f61c28dc2c8c23f93547", + "compat-results-cpu-0.11.2/4.native.txt": "491f772a829081eb99044315971a83009c0bbca4024a1379ef92f26ca0b4d896", + "compat-results-cpu-0.11.2/5.log": "cefb77df603453694f94428c2364aea36f9005b98a9d5fdbab84b4e6edbe454a", + "compat-results-cpu-0.11.2/5.native.txt": "224613ce042e01770207348c41ec7af1bf502d6375192c6b2ebfc5e15bdf564a", + "compat-results-cpu-0.9.2/1.log": "a0ed543f06276d8e9ff22d7949cd9f81763553dd63f1d8f0ea2b29f876dd0696", + "compat-results-cpu-0.9.2/2.log": "377f447838addd6deb60ee68082b8ef3c13d86787d4bff72712668ef40ec3f1a", + "compat-results-cpu-0.9.2/3.log": "f7440970185c1bdf94b55ef03ee5870c1fa1ae175cf1f0ea95bd3dbbfccedc1f", + "compat-results-cpu-0.9.2/4.log": "7c367bc27a879fe7319bc47bcd1c8ac650372b2eee7e0ac352401a64d0fb7499", + "compat-results-cpu-0.9.2/4.native.txt": "3593b7ff238985d5b1b9e3a7e6ad08f17be29a4e7e18e4c8850789405a1a8f40", + "compat-results-cpu-0.9.2/5.log": "98786f4a58cc14a0d2fd382b04d77be2fa93408bd9558557a7d35d2f30dd012a", + "compat-results-cpu-0.9.2/5.native.txt": "3ff1adeb6056b0b67b3212eeb90c388414ad32831a2e8c53497fa2dd141c366e" + }, + "fresh_resolver": { + "status": "pass", + "ignore_installed": true, + "prereleases_enabled": false, + "synthetic_test_version": "2026.10.4.2", + "note": "Synthetic local wheels only; no release/version choice or publication. Ordinary PyPI dependency resolution, no incident constraints. Earlier harness/index/disk failures retained in scratch.", + "selected": { + "jax": "0.9.2", + "jaxlib": "0.9.2", + "autolens": "2026.10.4.2", + "nautilus-sampler": "1.0.5", + "autogalaxy": "2026.10.4.2", + "autonerves": "2026.10.4.2", + "jaxnnls": "1.0.1", + "ml_dtypes": "0.6.0", + "numpy": "2.5.3", + "opt_einsum": "3.4.0", + "astropy": "8.0.1", + "autoarray": "2026.10.4.2", + "scipy": "1.17.1", + "autofit": "2026.10.4.2", + "astunparse": "1.6.3", + "corner": "2.2.2", + "dynesty": "2.1.5", + "gprof2dot": "2021.2.21", + "networkx": "3.1", + "psutil": "6.1.0", + "pyvis": "0.3.2", + "timeout-decorator": "0.5.0", + "jax_zero_contour": "2.0.0", + "PyYAML": "6.0.3", + "scikit-learn": "1.8.0", + "threadpoolctl": "3.7.0", + "typing-inspect": "0.9.0", + "anesthetic": "2.16.0", + "astropy-iers-data": "0.2026.9.28.0.59.37", + "decorator": "5.3.1", + "dill": "0.4.1", + "emcee": "3.1.6", + "h5py": "3.16.0", + "joblib": "1.6.0", + "matplotlib": "3.11.2", + "mypy_extensions": "1.1.0", + "numpydoc": "1.11.0", + "optax": "0.2.8", + "packaging": "26.3", + "pyerfa": "2.0.1.5", + "scikit-image": "0.26.0", + "SQLAlchemy": "2.0.54", + "typing_extensions": "4.16.0", + "xxhash": "3.4.1", + "array-api-compat": "1.15.0", + "tqdm": "4.70.1", + "absl-py": "2.5.0", + "cloudpickle": "3.1.2", + "contourpy": "1.4.0", + "cycler": "0.12.1", + "fonttools": "4.66.1", + "greenlet": "3.5.6", + "ImageIO": "2.38.0", + "ipython": "9.12.0", + "Jinja2": "3.1.6", + "jsonpickle": "4.1.3", + "kiwisolver": "1.5.1", + "lazy-loader": "0.6", + "pandas": "3.0.6", + "pillow": "12.3.0", + "pyparsing": "3.3.3", + "python-dateutil": "2.9.0.post0", + "six": "1.17.0", + "Sphinx": "9.1.0", + "tifffile": "2026.9.20", + "wheel": "0.48.0", + "alabaster": "1.0.0", + "babel": "2.18.0", + "docutils": "0.22.4", + "imagesize": "2.0.1", + "ipython_pygments_lexers": "1.1.1", + "jedi": "0.20.0", + "MarkupSafe": "3.0.4", + "matplotlib-inline": "0.2.2", + "pexpect": "4.9.0", + "prompt_toolkit": "3.0.53", + "Pygments": "2.21.0", + "requests": "2.34.2", + "roman-numerals": "4.1.0", + "snowballstemmer": "3.1.1", + "sphinxcontrib-applehelp": "2.0.0", + "sphinxcontrib-devhelp": "2.0.0", + "sphinxcontrib-htmlhelp": "2.1.0", + "sphinxcontrib-jsmath": "1.0.1", + "sphinxcontrib-qthelp": "2.0.0", + "sphinxcontrib-serializinghtml": "2.0.0", + "stack-data": "0.6.3", + "traitlets": "5.16.1", + "asttokens": "3.0.2", + "certifi": "2026.7.22", + "charset-normalizer": "3.5.2", + "executing": "2.2.1", + "idna": "3.20", + "parso": "0.8.7", + "ptyprocess": "0.7.0", + "urllib3": "2.8.0", + "wcwidth": "0.9.1", + "pure_eval": "0.2.4" + }, + "report_sha256": "0a909acaba32ac245d20b1aeeb28bec73ef0637923bfd384e7fc1e5307ecbbbc" + } +} diff --git a/diagnostics/release-37199991757-native.txt b/diagnostics/release-37199991757-native.txt new file mode 100644 index 0000000..aebcfde --- /dev/null +++ b/diagnostics/release-37199991757-native.txt @@ -0,0 +1,43 @@ +# Native witness: release diagnostic 37205459198 + +Source: https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37205459198 +Artifact: release-diagnostic-37205459198 / diagnostic-results/trial-2.native.txt +Raw native file SHA256: 1a025a43d895e247d7052ef0c2d84dfc33dafdd8fe92a4f1df726d39c42ced53 + +Trial1 PASS9.905s; trial2 TIMEOUT300.021s after2.6s compile. Both native tools exited0. +Python3.12.14, JAX/JAXlib0.10.2, four CPUs, XLA_FLAGS=--xla_cpu_multi_thread_eigen=false. +Four Eigen workers wait in LAPACK Cholesky batch fan-out; no FftThunk or ducc0 frames. + +Below: thread headings and selected frames, signatures and runner paths trimmed. + +Thread 28 (Thread 0x7ff0f93536c0 (LWP 3258) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#23 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) + +Thread 27 (Thread 0x7ff0f8b526c0 (LWP 3259) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#23 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) + +Thread 26 (Thread 0x7ff0f3fff6c0 (LWP 3260) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#24 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) + +Thread 25 (Thread 0x7ff0f37fe6c0 (LWP 3261) "tf_XLAEigen"): +#6 0x00007ff10e7eb2d1 in absl::lts_20260107::BlockingCounter::Wait() +#7 0x00007ff10e6f642e in jax::ParallelBatchMap(xla::ffi::ThreadPool, long, long, absl::lts_20260107::FunctionRef) +#8 0x00007ff10e713ce5 in jax::CholeskyFactorization<(xla::ffi::DataType)12, int>::Kernel(xla::ffi::ThreadPool, xla::ffi::Buffer<(xla::ffi::DataType)12, 18446744073709551615ul>, ja [signature trimmed] +#11 0x00007ff10e7380a9 in lapack_dpotrf_ffi +#15 0x00007ff11735356e in xla::cpu::CustomCallThunk::Execute(xla::cpu::Thunk::ExecuteParams const&) +#23 0x00007ff1161afb44 in Eigen::ThreadPoolTempl::WorkerLoop(int) diff --git a/diagnostics/release-37199991757.json b/diagnostics/release-37199991757.json new file mode 100644 index 0000000..a1d7855 --- /dev/null +++ b/diagnostics/release-37199991757.json @@ -0,0 +1,140 @@ +{ + "candidate_jax": "0.11.2", + "hands_sha": "760affdbcbf0605266e49b33a65f7bef89f9b0b3", + "library_shas": { + "PyAutoArray": "1cbabf9a44bfb98925e35954c263d393b7affc9d", + "PyAutoFit": "035ef1c8f79e4d488bb842c926e7d75c976cf8b6", + "PyAutoGalaxy": "4675fb265dd4c6d40aea8f809285b6d14dfb6325", + "PyAutoLens": "f9cff6b5c918748de131aade02aa864f17bfa0eb", + "PyAutoNerves": "d7e9f467da187e885139a5adb16baefbc8df59a7" + }, + "limitations": [ + "Package versions recovered from successful pip installation log lines; original wheel hashes and runner native stacks were not recorded.", + "The original runner OS image and CPU hardware are not pinned. Record the diagnostic topology; passing repetitions do not clear release validation.", + "Dataset is regenerated by the unchanged workspace bootstrap; original generated FITS files were not retained in the failed artifact." + ], + "packages": { + "MarkupSafe": "3.0.4", + "PyYAML": "6.0.3", + "SQLAlchemy": "2.0.54", + "absl-py": "2.5.0", + "alabaster": "1.0.0", + "anesthetic": "2.16.0", + "array_api_compat": "1.15.0", + "astropy": "8.0.1", + "astropy-iers-data": "0.2026.9.28.0.59.37", + "asttokens": "3.0.2", + "astunparse": "1.6.3", + "attrs": "26.1.0", + "autoarray": "2026.10.4.1.dev80201", + "autofit": "2026.10.4.1.dev80201", + "autogalaxy": "2026.10.4.1.dev80201", + "autolens": "2026.10.4.1.dev80201", + "autonerves": "2026.10.4.1.dev80201", + "babel": "2.18.0", + "blackjax": "1.7.1", + "certifi": "2026.7.22", + "charset_normalizer": "3.5.2", + "cloudpickle": "3.1.2", + "contourpy": "1.4.0", + "coolest": "0.1.11", + "corner": "2.2.2", + "cycler": "0.12.1", + "decorator": "5.3.1", + "dill": "0.4.1", + "dm-tree": "0.1.10", + "docutils": "0.22.4", + "dynesty": "2.1.5", + "emcee": "3.1.6", + "executing": "2.2.1", + "fonttools": "4.66.1", + "gast": "0.7.0", + "getdist": "1.4", + "gprof2dot": "2021.2.21", + "greenlet": "3.5.6", + "h5py": "3.16.0", + "idna": "3.20", + "imageio": "2.38.0", + "imagesize": "2.0.1", + "iniconfig": "2.3.0", + "ipython": "9.12.0", + "ipython-pygments-lexers": "1.1.1", + "jax": "0.10.2", + "jax_zero_contour": "2.0.0", + "jaxlib": "0.10.2", + "jaxnnls": "1.0.1", + "jedi": "0.20.0", + "jinja2": "3.1.6", + "joblib": "1.6.0", + "jsonpickle": "4.1.3", + "kiwisolver": "1.5.1", + "lazy-loader": "0.6", + "llvmlite": "0.50.0", + "matplotlib": "3.11.2", + "matplotlib-inline": "0.2.2", + "ml_dtypes": "0.6.0", + "mypy-extensions": "1.1.0", + "nautilus-sampler": "1.0.5", + "networkx": "3.1", + "nufftax": "0.6.1", + "numba": "0.68.0", + "numpy": "2.5.3", + "numpydoc": "1.11.0", + "opt_einsum": "3.4.0", + "optax": "0.2.8", + "packaging": "26.3", + "pandas": "3.0.6", + "parso": "0.8.7", + "pexpect": "4.9.0", + "pillow": "12.3.0", + "pluggy": "1.6.0", + "prompt_toolkit": "3.0.53", + "psutil": "6.1.0", + "ptyprocess": "0.7.0", + "pure-eval": "0.2.4", + "pyerfa": "2.0.1.5", + "pygments": "2.21.0", + "pyparsing": "3.3.3", + "pytest": "9.1.1", + "python-dateutil": "3.9.0", + "pyvis": "0.3.2", + "requests": "2.34.2", + "roman-numerals": "4.1.0", + "scikit-image": "0.26.0", + "scikit-learn": "1.8.0", + "scipy": "1.17.1", + "seaborn": "0.13.2", + "setuptools": "84.0.0", + "six": "1.17.0", + "snowballstemmer": "3.1.1", + "sphinx": "9.1.0", + "sphinxcontrib-applehelp": "2.0.0", + "sphinxcontrib-devhelp": "2.0.0", + "sphinxcontrib-htmlhelp": "2.1.0", + "sphinxcontrib-jsmath": "1.0.1", + "sphinxcontrib-qthelp": "2.0.0", + "sphinxcontrib-serializinghtml": "2.0.0", + "stack_data": "0.6.3", + "tfp-nightly": "0.26.0.dev20260713", + "threadpoolctl": "3.7.0", + "tifffile": "2026.9.20", + "timeout-decorator": "0.5.0", + "tqdm": "4.70.1", + "traitlets": "5.16.1", + "typing-extensions": "4.16.0", + "typing-inspect": "0.9.0", + "urllib3": "2.8.0", + "wcwidth": "0.9.1", + "wheel": "0.48.0", + "wrapt": "2.5.0", + "xxhash": "3.4.1", + "zeus-mcmc": "2.5.4" + }, + "python": "3.12.14", + "schema_version": 1, + "script": "multi_dataset/jax_likelihood/rectangular_rtu.py", + "source_job": "integrate / run_scripts (3.12, autolens_test, multi_dataset)", + "source_run": "https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37199991757", + "workspace_repo": "PyAutoLabs/autolens_workspace_test", + "workspace_sha": "df35fd4553f861575af7c27ad25b5f4d8d97d09c" +} diff --git a/docs/release_validation.md b/docs/release_validation.md index ab600f5..cd8d1ac 100644 --- a/docs/release_validation.md +++ b/docs/release_validation.md @@ -195,3 +195,97 @@ dispatching the `release-integrate.yml` channel), an ingested rehearsal-only report still (correctly) gates YELLOW: the source was built and TestPyPI-installed, but not yet exercised at release fidelity. `mode: release` is what supplies the `integrate` stage that flips this to GREEN-eligible. +# Bounded investigation of a failed wheel run + +`release-diagnostic.yml` investigates the saved `rectangular_rtu.py` timeout +from run `37199991757`. It is dispatched manually +with `repeats` of 2, 4 or 6 total trials. It does not run the integration matrix. + +The manifest `diagnostics/release-37199991757.json` preserves the final 115 +package versions from the failed installation log, the exact workspace and +Hands commits, Python 3.12.14 and the rehearsed library SHAs. In particular, +the failed release workflow downgraded JAX/JAXlib to 0.10.2, while the earlier +passing source retimes used 0.11.2. This difference is a confound to test, +not a demonstrated cause. + +The runner resolves the pinned workspace's release profile through pinned +Hands, verifies package versions and wheel import origins, and executes only +the affected script in fresh processes. It captures `/proc` thread states, +`py-spy --native` and GDB stacks at 120 seconds, terminates at 300 seconds, +and stops on any candidate failure. The comparison interleaves original 0.10.2 +controls and 0.11.2 candidates on the same runner and dataset, changing only +JAX/JAXlib in separate virtual environments. Original control timeouts remain +in the artifacts; a successful comparison requires a native-confirmed Cholesky +pool stall in at least one control and every candidate passing. Passing controls +alone are inconclusive, not evidence of a remedy. Diagnostic artifacts preserve the manifest, +pip installation URLs/hashes, actual runtime and environment provenance, +stdout/stderr, native capture failures, results and newly generated FITS data. +The original runner image, hardware, wheel hashes and generated FITS data +were not retained; the receipt exposes these reproduction limits. + +These are **diagnostic results only**. Neither passing repetitions nor a green +workflow clears the failed release. This workflow emits no release stage +report and never calls validation ingest. Do not raise caps, quarantine the +script, or identify a causal repair without reproducible or native evidence. + +The first exact hosted replay, [37205459198](https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37205459198), passed once in 9.905s and then stalled after 2.6s compilation. Both native tools captured all four Eigen workers waiting in `BlockingCounter::Wait` through `ParallelBatchMap`, `CholeskyFactorization` and `lapack_dpotrf_ffi`. The committed [native witness](../diagnostics/release-37199991757-native.txt) includes the raw artifact's SHA256. There are no FFT/ducc0 frames. + +[JAX 0.10.2](https://github.com/jax-ml/jax/blob/jax-v0.10.2/jaxlib/cpu/lapack_kernels.cc) schedules LAPACK batch chunks to the thread pool and blocks waiting for them; [JAX 0.11.2](https://github.com/jax-ml/jax/blob/jax-v0.11.2/jaxlib/cpu/lapack_kernels.cc) compiles this parallel path out of open-source builds. This is the source-grounded candidate tested by the comparison, distinct from the historical FFT workaround. No upstream report was filed. + +The interleaved comparison [37206724174](https://github.com/PyAutoLabs/PyAutoHeart/actions/runs/37206724174) passed all three controls (15.724/12.671/12.733s) and all three candidates (12.359/12.668/12.167s). It correctly returned **inconclusive** because the control did not stall on that runner. This does not establish a measured failure-rate improvement. The earlier exact replay supplies the native-confirmed failure; the source change removes that captured blocking path, and the candidate executions verify this script with the same release wheels. + +The smoke, integration and notebook dependency recipes now require matching JAX/JAXlib `>=0.11.2,<0.12`, preventing the old `<0.11` override from downgrading into the captured LAPACK deadlock. The original 115-package diagnostic control remains pinned to 0.10.2. No timeout, test selection or release-readiness rule changes. Full release integration has not been repeated: the failed validation remains authoritative until a separately authorized post-merge validation succeeds. + +### User-facing compatibility policy (2026-10-04) + +The package repair preserves `>=0.7,<0.12` while excluding `0.10.*` and `0.11.0`. +The LAPACK parallel batch path first appears in tagged 0.10.0 source and is disabled +in open-source builds from 0.11.1; 0.9.2 lacks the path. Only 0.10.2 was observed +deadlocking here. Exclusions of the other releases are source-based precautions. +The two diagnostic endpoints are 0.9.2 and 0.11.2, not a new blanket minimum of +0.11.2 or certification of every permitted older version. + +Nerves owns the exclusions and retains its Intel-macOS marker. Each repaired +Fit/Array/Galaxy/Lens/CTI package directly requires `autonerves>2026.10.4.1` to +prevent resolver fallback to an older, permissive Nerves. Hands' three explicit +JAX installs share the exclusions. Heart's integration baseline remains the +0.11.2 series; `jax-compatibility.yml` exercises both endpoints against the saved +wheel stack and original likelihood, plus numerical/sampler and historical FFT +witnesses. The original incident comparison is now manual-only to avoid an +unnecessary old-runtime replay on every PR update. + +**Release ordering:** merge/publish the policy-bearing Nerves before the repaired +family wheels. The strict bound rejects the old release's `.devN` and `.postN` +variants too; a same-day rehearsal must use a higher base version chosen by the +human release process. Source CI builds use the existing `setup.py` development +version `9999.0.0.dev0` and satisfy the guard. None of this retroactively repairs +published wheels or prevents an unconstrained installer from selecting an entirely +older family. Existing lockfiles require an explicit repaired-family upgrade. + +[Local receipts](../diagnostics/jax-compatibility-20261004.json) record: + +- Original rectangular likelihood, including its existing JIT/vmap assertions: + three CPU passes and one actual CUDA pass per version, with the same incident + wheels, 113 non-JAX pins and saved dataset. Local Python is 3.12.10, not the + incident's 3.12.14; the hosted workflow pins 3.12.14. +- Imaging light-profile gradient checks pass for both versions against finite + differences. Independent regularized-likelihood/Cholesky, gradient, NUFFT/direct + DFT, Optax optimizer and short BlackJAX NUTS execution checks pass on CPU and GPU. + The older endpoint also passes with NumPy2.0.0/SciPy1.13.0. The short NUTS run + tests finite execution, not posterior convergence. +- The historical FFT reproducer completes 20 iterations on both endpoints with + `--xla_cpu_multi_thread_eigen=false`. The separate FFT workaround is retained. +- All six real built wheels have the intended metadata. 25 live resolver cases + across the five consumer packages accept both endpoints and reject 0.10.2, + 0.11.0 and old Nerves, with older published candidates still available. A fresh `--ignore-installed` + resolver run without prereleases also selects the protected family and JAX0.9.2 + using synthetic stable local wheels (no release or version selection). + +**Incomplete broader check:** `point_source/jax_grad/gradient.py` completed its +solved-source finite-difference checks on both versions, then exceeded the 300s +local diagnostic cap in the later portion on both. Logs/native-capture attempts +and hashes are retained. This does not establish a version-specific regression +or its cause, and is not a passing result. No cap or test selection was changed +in production. Local timings were collected with other validation running and +must not be interpreted as a performance comparison. Full release validation +remains failed and must be assessed separately after human merge/authorization. diff --git a/tests/test_release_diagnostic.py b/tests/test_release_diagnostic.py new file mode 100644 index 0000000..603b465 --- /dev/null +++ b/tests/test_release_diagnostic.py @@ -0,0 +1,269 @@ +"""The diagnostic must fail closed and capture the live child before killing it.""" + +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location( + "release_diagnostic", ROOT / ".github/scripts/release_diagnostic.py" +) +diag = importlib.util.module_from_spec(spec) +spec.loader.exec_module(diag) + + +def manifest(): + return diag.load_manifest(ROOT / "diagnostics/release-37199991757.json") + + +@pytest.mark.parametrize( + "key,value", + [ + ("workspace_sha", "main"), + ("hands_sha", "../bad"), + ("python", "3.12"), + ("script", "../../outside.py"), + ("script", "/tmp/outside.py"), + ("workspace_repo", "../repo"), + ("packages", {}), + ("schema_version", 2), + ], +) +def test_rejects_unpinned_or_escaping_manifest(tmp_path, key, value): + data = manifest() + data[key] = value + path = tmp_path / "manifest.json" + path.write_text(json.dumps(data)) + with pytest.raises(ValueError): + diag.load_manifest(path) + + +def test_prepared_pins_retain_failed_runtime_and_no_release_artifact(tmp_path): + data = manifest() + diag.prepare(data, tmp_path) + requirements = (tmp_path / "requirements.txt").read_text() + assert "jax==0.10.2\n" in requirements + assert "jaxlib==0.10.2\n" in requirements + assert len(requirements.splitlines()) == 115 + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "manifest.json", + "requirements.txt", + ] + + +def test_runtime_and_source_shadowing_fail_provenance(monkeypatch, tmp_path): + data = manifest() + monkeypatch.setattr(diag.subprocess, "check_output", lambda *a, **kw: "0" * 40) + monkeypatch.setattr(diag.platform, "python_version", lambda: "3.12.10") + monkeypatch.setattr( + diag.importlib.metadata, "version", lambda name: data["packages"][name] + ) + monkeypatch.setattr( + diag.importlib.util, + "find_spec", + lambda name: type("Spec", (), {"origin": f"/source/{name}/__init__.py"})(), + ) + receipt = diag.provenance(data, tmp_path, tmp_path) + assert receipt["diagnostic_only"] + assert "Python patch version differs from manifest" in receipt["errors"] + assert "workspace commit differs from manifest" in receipt["errors"] + assert any("autolens is not imported" in error for error in receipt["errors"]) + + +def test_live_dump_precedes_timeout_and_child_is_reaped(tmp_path): + captured = [] + + def dump(pid, output): + os.kill(pid, 0) # must still exist when sampled + captured.append(pid) + output.write_text("live stack witness") + + result = diag.trial( + [ + sys.executable, + "-c", + "import time; print('started', flush=True); time.sleep(20)", + ], + tmp_path, + os.environ.copy(), + tmp_path / "trial.log", + cap=0.3, + dump_after=0.1, + dump=dump, + ) + assert result["status"] == "timeout" + assert result["elapsed_s"] < 3 + assert "started" in (tmp_path / "trial.log").read_text() + assert (tmp_path / "trial.native.txt").read_text() == "live stack witness" + with pytest.raises(ProcessLookupError): + os.kill(captured[0], 0) + + +def test_failure_exit_and_output_are_not_reported_as_pass(tmp_path): + result = diag.trial( + [sys.executable, "-c", "print('adverse evidence'); raise SystemExit(7)"], + tmp_path, + os.environ.copy(), + tmp_path / "failed.log", + cap=3, + dump_after=1, + ) + assert result["status"] == "fail" + assert result["returncode"] == 7 + assert "adverse evidence" in (tmp_path / "failed.log").read_text() + + +def test_native_helper_failure_is_preserved(monkeypatch, tmp_path): + def unavailable(*args, **kwargs): + raise FileNotFoundError("diagnostic helper unavailable") + + monkeypatch.setattr(subprocess, "run", unavailable) + output = tmp_path / "native.txt" + diag.native_dump(os.getpid(), output) + assert output.read_text().count("diagnostic helper unavailable") == 2 + + +def test_repeats_cannot_exceed_bounded_budget(tmp_path): + with pytest.raises(SystemExit) as exc: + diag.main( + [ + "prepare", + "--manifest", + str(ROOT / "diagnostics/release-37199991757.json"), + "--output", + str(tmp_path), + "--repeats", + "7", + ] + ) + assert exc.value.code == 2 + + +@pytest.mark.parametrize( + "control,candidate,signature,expected", + [ + ("timeout", "pass", True, "captured-stall-avoided"), + ("timeout", "pass", False, "inconclusive"), + ("pass", "pass", False, "inconclusive"), + ("timeout", "timeout", True, "candidate-failed"), + ("fail", "pass", False, "control-failed"), + ], +) +def test_comparison_requires_native_control_and_passing_candidate( + control, candidate, signature, expected +): + rows = [ + {"arm": "control", "status": control, "cholesky_pool_stall": signature}, + {"arm": "candidate", "status": candidate}, + ] + assert diag.comparison_verdict(rows, 2) == expected + partial = ( + expected if expected in {"candidate-failed", "control-failed"} else "incomplete" + ) + assert diag.comparison_verdict(rows, 4) == partial + assert diag.comparison_verdict(list(reversed(rows)), 2) == "invalid-order" + + +def test_comparison_interleaves_and_preserves_control_failures_and_venv_path( + tmp_path, monkeypatch +): + from argparse import Namespace + + python = tmp_path / "candidate-python" + python.symlink_to(sys.executable) + args = Namespace( + candidate_python=python, + repeats=6, + manifest=ROOT / "diagnostics/release-37199991757.json", + workspace=tmp_path, + hands=tmp_path, + ) + commands = [] + + def run(command, **kwargs): + commands.append(command) + candidate = "--candidate" in command + directory = Path(command[command.index("--output") + 1]) + directory.mkdir() + result = { + "status": "pass" if candidate else "timeout", + "returncode": 0 if candidate else -9, + } + (directory / "diagnostic-results.json").write_text( + json.dumps({"trials": [result]}) + ) + if not candidate: + (directory / "trial-1.native.txt").write_text( + "BlockingCounter::Wait ParallelBatchMap CholeskyFactorization\n" * 4 + ) + return subprocess.CompletedProcess(command, 0 if candidate else 1) + + monkeypatch.setattr(diag.subprocess, "run", run) + assert diag.compare(args, manifest(), tmp_path) == 0 + assert ["--candidate" in command for command in commands] == [False, True] * 3 + assert all(command[0] == str(python.absolute()) for command in commands[1::2]) + report = json.loads((tmp_path / "comparison.json").read_text()) + assert [row["status"] for row in report["trials"]] == ["timeout", "pass"] * 3 + with pytest.raises(ValueError, match="preserve previous evidence"): + diag.compare(args, manifest(), tmp_path) + + +def test_candidate_preparation_changes_only_the_runtime_pair(tmp_path): + diag.main( + [ + "prepare", + "--candidate", + "--manifest", + str(ROOT / "diagnostics/release-37199991757.json"), + "--output", + str(tmp_path), + ] + ) + actual = json.loads((tmp_path / "manifest.json").read_text()) + original = manifest() + changed = { + key + for key in original["packages"] + if actual["packages"][key] != original["packages"][key] + } + assert changed == {"jax", "jaxlib"} + assert actual["packages"]["jax"] == actual["packages"]["jaxlib"] == "0.11.2" + assert actual["workspace_sha"] == original["workspace_sha"] + + +def test_early_candidate_failure_has_adverse_durable_verdict(tmp_path, monkeypatch): + from argparse import Namespace + + args = Namespace( + candidate_python=Path(sys.executable), + repeats=6, + manifest=ROOT / "diagnostics/release-37199991757.json", + workspace=tmp_path, + hands=tmp_path, + ) + + def run(command, **kwargs): + candidate = "--candidate" in command + directory = Path(command[command.index("--output") + 1]) + directory.mkdir() + result = { + "status": "fail" if candidate else "pass", + "returncode": 7 if candidate else 0, + } + (directory / "diagnostic-results.json").write_text( + json.dumps({"trials": [result]}) + ) + return subprocess.CompletedProcess(command, 1 if candidate else 0) + + monkeypatch.setattr(diag.subprocess, "run", run) + assert diag.compare(args, manifest(), tmp_path) == 1 + report = json.loads((tmp_path / "comparison.json").read_text()) + assert report["verdict"] == "candidate-failed" + assert len(report["trials"]) == 2 + assert report["trials"][1]["returncode"] == 7