| Path | ' in html
+ assert "copyText" in html
+ assert "node.display_path" in html
+
+
+def test_draw_dag_cli_save_as(tmp_path, monkeypatch):
+ """Verify draw_dag --save-as auto-detects extensions and saves without launching server."""
+ full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()}
+ dependencies = {"j_train": ["j_root"]}
+ job_tags_dict = {"j_train": {"lr": 0.001}}
+
+ run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000"
+ run_dir.mkdir(parents=True)
+
+ monkeypatch.setattr(
+ draw_dag,
+ "resolve_run_dir",
+ lambda ws, exp_id, run_id: ("20260101_120000", run_dir),
+ )
+ monkeypatch.setattr(
+ draw_dag,
+ "load_run_graph",
+ lambda r_dir: (full_jobs, dependencies, job_tags_dict),
+ )
+
+ server_called = []
+ monkeypatch.setattr(
+ draw_dag,
+ "serve_dag_web",
+ lambda *args, **kwargs: server_called.append(True),
+ )
+
+ # 1. Test --save-as with .html: saves HTML and does NOT launch server even if --web was passed
+ html_out = tmp_path / "subfolder" / "dag.html"
+ monkeypatch.setattr(
+ "sys.argv",
+ ["draw_dag", "--web", "--save-as", str(html_out)],
+ )
+ draw_dag.main()
+ assert html_out.exists()
+ assert "" in html_out.read_text()
+ assert len(server_called) == 0, (
+ "Server should not be launched when --save-as ends in .html"
+ )
+
+ # 2. Test --save-as with .json
+ json_out = tmp_path / "subfolder" / "dag.json"
+ monkeypatch.setattr(
+ "sys.argv",
+ ["draw_dag", "--save-as", str(json_out)],
+ )
+ draw_dag.main()
+ assert json_out.exists()
+ assert "j_train" in json_out.read_text()
+
+
+def test_web_dag_template_loading():
+ """Verify that the dedicated HTML template file loads correctly."""
+ from experimaestro.tools.xpm_dag.xpm_web_dag import (
+ HTML_TEMPLATE,
+ _load_html_template,
+ )
+
+ loaded = _load_html_template()
+ assert len(loaded) > 0
+ assert loaded == HTML_TEMPLATE
+ assert "" in loaded
+ assert "__TITLE__" in loaded
+ assert "__RUN_ID__" in loaded
+ assert "__DEFAULT_LAYOUT__" in loaded
+ assert "__GRAPH_DATA__" in loaded
+
+
+def test_cli_dag_help():
+ """Verify experimaestro dag --help and experimaestro experiments dag --help."""
+ from click.testing import CliRunner
+ from experimaestro.cli import cli
+
+ runner = CliRunner()
+ res1 = runner.invoke(cli, ["dag", "--help"])
+ assert res1.exit_code == 0
+ assert "Visualize and export experiment dependency DAGs" in res1.output
+ assert "--web" in res1.output
+ assert "--save-as" in res1.output
+ assert "--list-configs" in res1.output
+
+ res2 = runner.invoke(cli, ["experiments", "dag", "--help"])
+ assert res2.exit_code == 0
+ assert "Visualize and export experiment dependency DAGs" in res2.output
+
+
+def test_cli_dag_save_as_and_json(tmp_path, monkeypatch):
+ """Verify experimaestro dag --save-as and --json via Click CLI."""
+ from click.testing import CliRunner
+ from experimaestro.cli import cli
+ from experimaestro.tools.xpm_dag import draw_dag
+
+ full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()}
+ dependencies = {"j_train": ["j_root"]}
+ job_tags_dict = {"j_train": {"lr": 0.001}}
+
+ run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000"
+ run_dir.mkdir(parents=True)
+
+ monkeypatch.setattr(
+ draw_dag,
+ "resolve_run_dir",
+ lambda ws, exp_id, run_id: ("20260101_120000", run_dir),
+ )
+ monkeypatch.setattr(
+ draw_dag,
+ "load_run_graph",
+ lambda r_dir: (full_jobs, dependencies, job_tags_dict),
+ )
+
+ runner = CliRunner()
+
+ # 1. Test --save-as html
+ html_out = tmp_path / "cli_out.html"
+ res_html = runner.invoke(cli, ["dag", "test_exp", "--save-as", str(html_out)])
+ assert res_html.exit_code == 0
+ assert html_out.exists()
+ assert "" in html_out.read_text()
+
+ # 2. Test --json stdout
+ res_json = runner.invoke(cli, ["dag", "test_exp", "--json"])
+ assert res_json.exit_code == 0
+ parsed = json.loads(res_json.output)
+ assert "nodes" in parsed
+ assert any(n["id"] == "j_train" for n in parsed["nodes"])
+
+
+def test_resolve_run_dir_auto_detect(tmp_path):
+ """Verify resolve_run_dir auto-selects when a single experiment exists."""
+ from experimaestro.tools.xpm_dag.xpm_dag import resolve_run_dir
+
+ ws_dir = tmp_path / "workspace"
+ exp_dir = ws_dir / "experiments" / "single_experiment" / "20260101_120000"
+ exp_dir.mkdir(parents=True)
+
+ # 1. Single experiment exists: auto-selects without passing experiment_id
+ res = resolve_run_dir(workspace_arg=str(ws_dir), experiment_id=None)
+ assert res is not None
+ run_id, resolved_dir = res
+ assert run_id == "20260101_120000"
+ assert resolved_dir == exp_dir
+
+ # 2. Multiple experiments exist: returns None when experiment_id is omitted
+ exp2_dir = ws_dir / "experiments" / "another_experiment" / "20260101_130000"
+ exp2_dir.mkdir(parents=True)
+ res_multi = resolve_run_dir(workspace_arg=str(ws_dir), experiment_id=None)
+ assert res_multi is None
+
+
+def test_cli_dag_defaults_to_web(tmp_path, monkeypatch):
+ """Verify experimaestro dag defaults to launching the web viewer."""
+ from click.testing import CliRunner
+ from experimaestro.cli import cli
+ from experimaestro.tools.xpm_dag import draw_dag
+
+ full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()}
+ dependencies = {"j_train": ["j_root"]}
+ job_tags_dict = {"j_train": {"lr": 0.001}}
+
+ run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000"
+ run_dir.mkdir(parents=True)
+
+ monkeypatch.setattr(
+ draw_dag,
+ "resolve_run_dir",
+ lambda ws, exp_id, run_id: ("20260101_120000", run_dir),
+ )
+ monkeypatch.setattr(
+ draw_dag,
+ "load_run_graph",
+ lambda r_dir: (full_jobs, dependencies, job_tags_dict),
+ )
+
+ server_called = []
+ monkeypatch.setattr(
+ draw_dag,
+ "serve_dag_web",
+ lambda *args, **kwargs: server_called.append(True),
+ )
+
+ runner = CliRunner()
+ # Calling dag with NO flags: should default to web view
+ res = runner.invoke(cli, ["dag", "test_exp"])
+ assert res.exit_code == 0
+ assert len(server_called) == 1, "Web server should be launched by default"
+
+
+def test_cli_dag_optional_tree(tmp_path, monkeypatch):
+ """Verify experimaestro dag --tree prints text tree and does NOT launch server."""
+ from click.testing import CliRunner
+ from experimaestro.cli import cli
+ from experimaestro.tools.xpm_dag import draw_dag
+
+ full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()}
+ dependencies = {"j_train": ["j_root"]}
+ job_tags_dict = {"j_train": {"lr": 0.001}}
+
+ run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000"
+ run_dir.mkdir(parents=True)
+
+ monkeypatch.setattr(
+ draw_dag,
+ "resolve_run_dir",
+ lambda ws, exp_id, run_id: ("20260101_120000", run_dir),
+ )
+ monkeypatch.setattr(
+ draw_dag,
+ "load_run_graph",
+ lambda r_dir: (full_jobs, dependencies, job_tags_dict),
+ )
+
+ server_called = []
+ monkeypatch.setattr(
+ draw_dag,
+ "serve_dag_web",
+ lambda *args, **kwargs: server_called.append(True),
+ )
+
+ runner = CliRunner()
+ res = runner.invoke(cli, ["dag", "test_exp", "--tree"])
+ assert res.exit_code == 0
+ assert len(server_called) == 0, (
+ "Web server should not be launched when --tree is given"
+ )
+ assert "DummyTask" in res.output
+ assert "j_root" in res.output
+
+
+def test_matplotlib_lazy_import(tmp_path, monkeypatch):
+ """Verify that running web view or --tree does not import matplotlib."""
+ import sys
+ from click.testing import CliRunner
+ from experimaestro.cli import cli
+ from experimaestro.tools.xpm_dag import draw_dag
+
+ full_jobs = {"j_root": DummyTask()}
+ dependencies = {}
+ job_tags_dict = {}
+
+ run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000"
+ run_dir.mkdir(parents=True)
+
+ monkeypatch.setattr(
+ draw_dag,
+ "resolve_run_dir",
+ lambda ws, exp_id, run_id: ("20260101_120000", run_dir),
+ )
+ monkeypatch.setattr(
+ draw_dag,
+ "load_run_graph",
+ lambda r_dir: (full_jobs, dependencies, job_tags_dict),
+ )
+
+ # Ensure matplotlib is not in sys.modules
+ sys.modules.pop("matplotlib", None)
+ sys.modules.pop("matplotlib.pyplot", None)
+
+ runner = CliRunner()
+ res = runner.invoke(cli, ["dag", "test_exp", "--tree"])
+ assert res.exit_code == 0
+ assert "matplotlib" not in sys.modules
+ assert "matplotlib.pyplot" not in sys.modules
+
+
+def test_draw_dag_depths_alignment_and_deep_deps():
+ """Verify that compute_dag_depths aligns same-stage tasks and pushes deeper dependencies."""
+ import networkx as nx
+ from experimaestro.tools.xpm_dag.draw_dag import compute_dag_depths
+
+ def make_node(name: str):
+ return type(name, (), {})()
+
+ full_jobs = {
+ "dataset": make_node("MNISTLabelledImages"),
+ "extra_prep": make_node("PreprocessTask"),
+ "deep_step": make_node("DeepHelperTask"),
+ "learn_shallow": make_node("Learn"),
+ "learn_deep": make_node("Learn"),
+ "eval_shallow": make_node("Evaluate"),
+ "eval_deep": make_node("Evaluate"),
+ }
+
+ dag = nx.DiGraph()
+ for jid in full_jobs:
+ dag.add_node(jid)
+
+ # Shallow pipeline: dataset -> learn_shallow -> eval_shallow
+ dag.add_edge("dataset", "learn_shallow")
+ dag.add_edge("learn_shallow", "eval_shallow")
+
+ # Deep pipeline: dataset -> extra_prep -> deep_step -> learn_deep -> eval_deep
+ dag.add_edge("dataset", "extra_prep")
+ dag.add_edge("extra_prep", "deep_step")
+ dag.add_edge("deep_step", "learn_deep")
+ dag.add_edge("learn_deep", "eval_deep")
+
+ depths = compute_dag_depths(
+ dag,
+ full_jobs,
+ align_types=True,
+ aligned_class_names={"Learn", "Evaluate"},
+ )
+
+ assert depths["learn_shallow"] == depths["learn_deep"] == 3
+ assert depths["eval_shallow"] >= 4
+ assert depths["eval_deep"] >= 4
+
+ for u, v in dag.edges():
+ assert depths[v] > depths[u], f"Edge {u} -> {v} violated monotonicity"
+
+
+def test_draw_dag_filter_helpers(tmp_path):
+ """Verify tag parsing, config listing, and subgraph extraction."""
+ from experimaestro.tools.xpm_dag.draw_dag import (
+ filter_subgraph_by_config,
+ parse_filter_arg,
+ )
+
+ # 1. Test parse_filter_arg
+ parsed = parse_filter_arg("lr=0.001,kernel_size=3,name=cnn,flag=true")
+ assert parsed == {"lr": 0.001, "kernel_size": 3, "name": "cnn", "flag": "true"}
+
+ # 2. Test filter_subgraph_by_config
+ class MockObj:
+ pass
+
+ full_jobs = {"root": MockObj(), "j1": MockObj(), "j2": MockObj(), "j3": MockObj()}
+ dependencies = {
+ "j1": ["root"],
+ "j2": ["root", "j1"],
+ "j3": ["root"],
+ }
+ job_tags_dict = {
+ "j1": {"lr": 0.01, "layers": 2},
+ "j2": {"lr": 0.01, "layers": 2},
+ "j3": {"lr": 0.001, "layers": 3},
+ }
+
+ filtered_jobs, filtered_deps = filter_subgraph_by_config(
+ full_jobs,
+ dependencies,
+ job_tags_dict,
+ target_tags={"lr": 0.01, "layers": 2},
+ )
+
+ assert set(filtered_jobs.keys()) == {"root", "j1", "j2"}
+ assert "j3" not in filtered_jobs
+ assert filtered_deps["j1"] == ["root"]
+ assert filtered_deps["j2"] == ["root", "j1"]
+
+
+def test_resolve_run_dir_error_reporting(tmp_path, caplog):
+ """Verify that resolve_run_dir provides helpful error messages listing available experiments and runs."""
+ from experimaestro.tools.xpm_dag.xpm_dag import resolve_run_dir
+
+ ws_dir = tmp_path / "test_ws"
+ exp_dir = ws_dir / "experiments"
+ (exp_dir / "exp_alpha" / "20260101_120000").mkdir(parents=True)
+ (exp_dir / "exp_beta" / "20260102_150000").mkdir(parents=True)
+
+ with caplog.at_level(logging.ERROR):
+ # 1. Non-existent experiment
+ res = resolve_run_dir(str(ws_dir), "exp_gamma", None)
+ assert res is None
+ assert "Experiment 'exp_gamma' not found" in caplog.text
+ assert "'exp_alpha'" in caplog.text and "'exp_beta'" in caplog.text
+
+ caplog.clear()
+
+ # 2. Non-existent run
+ res = resolve_run_dir(str(ws_dir), "exp_alpha", "wrong_run_id")
+ assert res is None
+ assert "Run 'wrong_run_id' not found" in caplog.text
+ assert "'20260101_120000'" in caplog.text
+
+
+def test_resolve_run_dir_remote_ssh(monkeypatch, tmp_path):
+ """Verify that resolve_run_dir delegates to remote SSH fetcher when workspace is remote."""
+ from pathlib import Path
+ from experimaestro.settings import SSHSettings, WorkspaceSettings
+ from experimaestro.tools.xpm_dag.xpm_dag import resolve_run_dir
+
+ ssh = SSHSettings(host="cluster.example.org")
+ ws = WorkspaceSettings(
+ id="cluster_ws",
+ path=Path("/remote/workspace"),
+ ssh=ssh,
+ )
+
+ fake_run_dir = tmp_path / "cached_run"
+ fake_run_dir.mkdir()
+
+ called = []
+
+ def mock_fetch(remote_ws, exp_id, r_id):
+ called.append((remote_ws.id, exp_id, r_id))
+ return "20260101_120000", fake_run_dir
+
+ monkeypatch.setattr(
+ "experimaestro.tools.xpm_dag.xpm_dag.find_workspace",
+ lambda **kwargs: ws,
+ )
+ monkeypatch.setattr(
+ "experimaestro.utils.remote.find_remote_workspace_experiment_run",
+ mock_fetch,
+ )
+
+ resolved_id, run_d = resolve_run_dir(
+ "cluster_ws", "my_remote_exp", "20260101_120000"
+ )
+ assert resolved_id == "20260101_120000"
+ assert run_d == fake_run_dir
+ assert called == [("cluster_ws", "my_remote_exp", "20260101_120000")]
+
+
+def test_load_run_graph_without_importing_remote_modules(tmp_path):
+ """Verify load_run_graph builds the DAG from JSONL without importing remote modules or files."""
+ from experimaestro import Action, Config, Task
+ from experimaestro.tools.xpm_dag.xpm_dag import load_run_graph
+
+ run_dir = tmp_path / "remote_run"
+ run_dir.mkdir()
+
+ cfg_id = "a" * 64
+ task_id = "b" * 64
+ out_cfg_id = "c" * 64
+ eval_task_id = "d" * 64
+ action_id = "e" * 64
+
+ objects_data = [
+ {
+ "id": 1,
+ "module": "non_existent_remote_pkg.models",
+ "type": "RemoteEncoderConfig",
+ "typename": "non_existent_remote_pkg.models.RemoteEncoderConfig",
+ "identifier": cfg_id,
+ "file": "/lustre/fsn1/projects/rech/cdt/uqo57sk/code/cross-encoders/src/tests.py",
+ "fields": {
+ "hidden_dim": 768,
+ "lr": 0.0001,
+ "mode": {
+ "type": "enum",
+ "module": "non_existent_remote_pkg.enums",
+ "enum": "Mode",
+ "value": "FAST",
+ },
+ "data_path": {
+ "type": "path",
+ "base": "workspace",
+ "value": "data/corpus",
+ },
+ "generated_listener_path": {
+ "type": "path",
+ "value": f"/lustre/ws/jobs/non_existent_remote_pkg.tasks.learn/{task_id}/out/listener",
+ },
+ },
+ },
+ {
+ "id": 2,
+ "module": "non_existent_remote_pkg.tasks",
+ "type": "Learn",
+ "typename": "non_existent_remote_pkg.tasks.Learn",
+ "identifier": task_id,
+ "file": "/lustre/fsn1/projects/rech/cdt/uqo57sk/code/cross-encoders/src/training_utils.py",
+ "fields": {
+ "encoder": {"type": "python", "value": 1},
+ },
+ },
+ {
+ "id": 3,
+ "module": "non_existent_remote_pkg.models",
+ "type": "ParameterLoader",
+ "typename": "non_existent_remote_pkg.models.ParameterLoader",
+ "identifier": out_cfg_id,
+ "task": 2,
+ "fields": {},
+ },
+ {
+ "id": 4,
+ "module": "tests",
+ "type": "Evaluate",
+ "typename": "tests.Evaluate",
+ "identifier": eval_task_id,
+ "file": "/lustre/fsn1/projects/rech/cdt/uqo57sk/code/cross-encoders/src/tests.py",
+ "init-tasks": [3],
+ "fields": {
+ "model": {"type": "python", "value": 1},
+ },
+ },
+ {
+ "id": 5,
+ "module": "non_existent_remote_pkg.actions",
+ "type": "ExportHuggingFace",
+ "typename": "non_existent_remote_pkg.actions.ExportHuggingFace",
+ "identifier": action_id,
+ "fields": {
+ "task": {"type": "python", "value": 2},
+ },
+ },
+ ]
+
+ with (run_dir / "objects.jsonl").open("w") as f:
+ for obj in objects_data:
+ f.write(json.dumps(obj) + "\n")
+
+ jobs_data = [
+ {
+ "job_id": task_id,
+ "task_id": "non_existent_remote_pkg.tasks.learn",
+ "tags": {"lr": 0.0001, "hidden_dim": 768},
+ },
+ {
+ "job_id": eval_task_id,
+ "task_id": "tests.evaluate",
+ "tags": {"lr": 0.0001, "hidden_dim": 768},
+ },
+ ]
+ with (run_dir / "jobs.jsonl").open("w") as f:
+ for j in jobs_data:
+ f.write(json.dumps(j) + "\n")
+
+ status_data = {
+ "job_states": {task_id: "done", eval_task_id: "running"},
+ "actions": {action_id: {"description": "Export model"}},
+ }
+ (run_dir / "status.json").write_text(json.dumps(status_data))
+
+ res = load_run_graph(run_dir)
+ assert res is not None
+ full_jobs, dependencies, job_tags_dict = res
+
+ assert set(full_jobs.keys()) == {
+ cfg_id,
+ task_id,
+ out_cfg_id,
+ eval_task_id,
+ action_id,
+ }
+ assert isinstance(full_jobs[cfg_id], Config)
+ assert not isinstance(full_jobs[cfg_id], (Task, Action))
+ assert isinstance(full_jobs[task_id], Task)
+ assert isinstance(full_jobs[eval_task_id], Task)
+ assert isinstance(full_jobs[action_id], Action)
+
+ assert type(full_jobs[cfg_id]).__name__ == "RemoteEncoderConfig"
+ assert type(full_jobs[task_id]).__name__ == "Learn"
+ assert type(full_jobs[action_id]).__name__ == "ExportHuggingFace"
+
+ assert cfg_id in dependencies[task_id]
+ assert task_id not in dependencies.get(cfg_id, [])
+ assert task_id in dependencies[out_cfg_id]
+ assert out_cfg_id in dependencies[eval_task_id]
+ assert cfg_id in dependencies[eval_task_id]
+ assert task_id in dependencies[action_id]
+
+ assert job_tags_dict[cfg_id] == {"hidden_dim": 768, "lr": 0.0001}
+ assert job_tags_dict[task_id] == {"hidden_dim": 768, "lr": 0.0001}
+
+ dag_json = build_dag_json(full_jobs, dependencies, job_tags_dict, run_dir=run_dir)
+ assert len(dag_json["nodes"]) == 5
+ nodes_by_id = {n["id"]: n for n in dag_json["nodes"]}
+ assert nodes_by_id[out_cfg_id]["type"] == "init"
+ assert nodes_by_id[out_cfg_id]["status"] == "init"
+ assert nodes_by_id[out_cfg_id]["init"] is True
+
+
+def test_init_task_node_type_and_filtering(tmp_path):
+ """Verify init_task nodes are typed as LightweightTask ('init') and kept by filter_tasks_only."""
+ from experimaestro import LightweightTask
+ from experimaestro.tools.xpm_dag import (
+ filter_tasks_only,
+ is_init_task_node,
+ load_run_graph,
+ )
+
+ run_dir = tmp_path / "init_run"
+ run_dir.mkdir(parents=True)
+
+ init_id = "1" * 64
+ cfg_id = "2" * 64
+ task_id = "3" * 64
+
+ objects_data = [
+ {
+ "id": 1,
+ "module": "pkg",
+ "type": "HFModelInitFromID",
+ "identifier": init_id,
+ "fields": {},
+ },
+ {
+ "id": 2,
+ "module": "pkg",
+ "type": "StaticConfig",
+ "identifier": cfg_id,
+ "fields": {},
+ },
+ {
+ "id": 3,
+ "module": "pkg",
+ "type": "Evaluate",
+ "identifier": task_id,
+ "task": 3,
+ "init-tasks": [1],
+ "fields": {"cfg": {"type": "python", "value": 2}},
+ },
+ ]
+ with (run_dir / "objects.jsonl").open("w") as f:
+ for obj in objects_data:
+ f.write(json.dumps(obj) + "\n")
+ with (run_dir / "jobs.jsonl").open("w") as f:
+ f.write(
+ json.dumps({"job_id": task_id, "task_id": "pkg.evaluate", "tags": {}})
+ + "\n"
+ )
+
+ full_jobs, dependencies, job_tags_dict = load_run_graph(run_dir)
+ assert isinstance(full_jobs[init_id], LightweightTask)
+ assert is_init_task_node(full_jobs[init_id]) is True
+ assert is_init_task_node(full_jobs[cfg_id]) is False
+ assert is_init_task_node(full_jobs[task_id]) is False
+
+ filtered_jobs, filtered_deps = filter_tasks_only(full_jobs, dependencies)
+ assert set(filtered_jobs.keys()) == {init_id, task_id}
+ assert init_id in filtered_deps[task_id]
+
+
+def test_bottleneck_grid_config_detection(tmp_path):
+ """Verify get_experiment_configs detects core bottleneck grid configs when dataset fan-outs exist."""
+ run_dir = tmp_path / "bottleneck_run"
+ run_dir.mkdir(parents=True)
+
+ records = [
+ # Upstream index builders (dataset/index fan-out)
+ {
+ "job_id": "idx1",
+ "task_id": "xpmir.index.sparse.sparseretrieverindexbuilder",
+ "tags": {"index_documents": "doc_a"},
+ },
+ {
+ "job_id": "idx2",
+ "task_id": "xpmir.index.sparse.sparseretrieverindexbuilder",
+ "tags": {"index_documents": "doc_b"},
+ },
+ # 1st-stage evaluations
+ {
+ "job_id": "eval1_a",
+ "task_id": "xpmir.evaluation.evaluate",
+ "tags": {"dataset": "ds_a", "first_stage": "splade"},
+ },
+ {
+ "job_id": "eval1_b",
+ "task_id": "xpmir.evaluation.evaluate",
+ "tags": {"dataset": "ds_b", "first_stage": "splade"},
+ },
+ # Core Learner grid-search jobs (2 configs)
+ {
+ "job_id": "learn1",
+ "task_id": "xpm_torch.learner.learner",
+ "tags": {"base": "ettin", "lr": 1e-5, "seed": 1},
+ },
+ {
+ "job_id": "learn2",
+ "task_id": "xpm_torch.learner.learner",
+ "tags": {"base": "ettin", "lr": 7e-6, "seed": 2},
+ },
+ # 2nd-stage evaluations (Cartesian product: 2 learners x 2 datasets = 4 jobs)
+ {
+ "job_id": "eval2_1a",
+ "task_id": "xpmir.evaluation.evaluate",
+ "tags": {
+ "base": "ettin",
+ "lr": 1e-5,
+ "seed": 1,
+ "dataset": "ds_a",
+ "first_stage": "splade",
+ "checkpoint": "last",
+ },
+ },
+ {
+ "job_id": "eval2_1b",
+ "task_id": "xpmir.evaluation.evaluate",
+ "tags": {
+ "base": "ettin",
+ "lr": 1e-5,
+ "seed": 1,
+ "dataset": "ds_b",
+ "first_stage": "splade",
+ "checkpoint": "last",
+ },
+ },
+ {
+ "job_id": "eval2_2a",
+ "task_id": "xpmir.evaluation.evaluate",
+ "tags": {
+ "base": "ettin",
+ "lr": 7e-6,
+ "seed": 2,
+ "dataset": "ds_a",
+ "first_stage": "splade",
+ "checkpoint": "last",
+ },
+ },
+ {
+ "job_id": "eval2_2b",
+ "task_id": "xpmir.evaluation.evaluate",
+ "tags": {
+ "base": "ettin",
+ "lr": 7e-6,
+ "seed": 2,
+ "dataset": "ds_b",
+ "first_stage": "splade",
+ "checkpoint": "last",
+ },
+ },
+ ]
+
+ with (run_dir / "jobs.jsonl").open("w") as f:
+ for r in records:
+ f.write(json.dumps(r) + "\n")
+
+ configs = get_experiment_configs(run_dir)
+ assert configs == [
+ {"base": "ettin", "lr": 1e-5, "seed": 1},
+ {"base": "ettin", "lr": 7e-6, "seed": 2},
+ ]
+
+
+def test_stack_dag_nodes_parallel_chains():
+ """Verify stack_dag_nodes collapses parallel dataset chains while preserving Learner branches."""
+ from experimaestro import LightweightTask
+ from experimaestro.tools.xpm_dag import stack_dag_nodes
+
+ InitCls = type("HFModelInitFromID", (LightweightTask,), {})
+ IdxCls = type("SparseRetrieverIndexBuilder", (Task,), {})
+ EvalCls = type("Evaluate", (Task,), {})
+ LearnCls = type("Learner", (Task,), {})
+ LoaderCls = type("SimpleModuleLoader", (LightweightTask,), {})
+
+ full_jobs = {
+ "init_hf": InitCls(),
+ "idx_1": IdxCls(),
+ "idx_2": IdxCls(),
+ "idx_3": IdxCls(),
+ "eval1_1": EvalCls(),
+ "eval1_2": EvalCls(),
+ "eval1_3": EvalCls(),
+ "learn_a": LearnCls(),
+ "learn_b": LearnCls(),
+ "loader_a": LoaderCls(),
+ "loader_b": LoaderCls(),
+ "eval2_a1": EvalCls(),
+ "eval2_a2": EvalCls(),
+ "eval2_a3": EvalCls(),
+ "eval2_b1": EvalCls(),
+ "eval2_b2": EvalCls(),
+ "eval2_b3": EvalCls(),
+ }
+ dependencies = {
+ "idx_1": ["init_hf"],
+ "idx_2": ["init_hf"],
+ "idx_3": ["init_hf"],
+ "eval1_1": ["idx_1", "init_hf"],
+ "eval1_2": ["idx_2", "init_hf"],
+ "eval1_3": ["idx_3", "init_hf"],
+ "loader_a": ["learn_a"],
+ "loader_b": ["learn_b"],
+ "eval2_a1": ["loader_a", "eval1_1"],
+ "eval2_a2": ["loader_a", "eval1_2"],
+ "eval2_a3": ["loader_a", "eval1_3"],
+ "eval2_b1": ["loader_b", "eval1_1"],
+ "eval2_b2": ["loader_b", "eval1_2"],
+ "eval2_b3": ["loader_b", "eval1_3"],
+ }
+ job_tags_dict = {
+ "idx_1": {"index_documents": "d1"},
+ "idx_2": {"index_documents": "d2"},
+ "idx_3": {"index_documents": "d3"},
+ "eval1_1": {"dataset": "d1", "first_stage": "splade"},
+ "eval1_2": {"dataset": "d2", "first_stage": "splade"},
+ "eval1_3": {"dataset": "d3", "first_stage": "splade"},
+ "learn_a": {"lr": 1e-5, "seed": 1},
+ "learn_b": {"lr": 7e-6, "seed": 2},
+ "eval2_a1": {"lr": 1e-5, "seed": 1, "dataset": "d1", "first_stage": "splade"},
+ "eval2_a2": {"lr": 1e-5, "seed": 1, "dataset": "d2", "first_stage": "splade"},
+ "eval2_a3": {"lr": 1e-5, "seed": 1, "dataset": "d3", "first_stage": "splade"},
+ "eval2_b1": {"lr": 7e-6, "seed": 2, "dataset": "d1", "first_stage": "splade"},
+ "eval2_b2": {"lr": 7e-6, "seed": 2, "dataset": "d2", "first_stage": "splade"},
+ "eval2_b3": {"lr": 7e-6, "seed": 2, "dataset": "d3", "first_stage": "splade"},
+ }
+
+ s_jobs, s_deps, s_tags, s_members = stack_dag_nodes(
+ full_jobs, dependencies, job_tags_dict
+ )
+ # 17 nodes collapse into 9:
+ # init_hf (1), idx (x3), eval1 (x3), learn_a (1), learn_b (1), loader_a (1), loader_b (1), eval2_a (x3), eval2_b (x3)
+ assert len(s_jobs) == 9
+ assert "learn_a" in s_jobs and "learn_b" in s_jobs
+ assert len(s_members["idx_1"]) == 3
+ assert len(s_members["eval1_1"]) == 3
+ assert len(s_members["eval2_a1"]) == 3
+ assert len(s_members["eval2_b1"]) == 3
+ assert s_tags["eval2_a1"] == {
+ "lr": 1e-5,
+ "seed": 1,
+ "dataset": "<3 values>",
+ "first_stage": "splade",
+ }
+
+ dag_json = build_dag_json(full_jobs, dependencies, job_tags_dict)
+ html = generate_dag_html(dag_json)
+ assert 'id="btn-stack"' in html
+ assert "stack-backframe" in html
+ assert "stack-exterior-badge" in html
+ assert "type-init" in html
+
+
+def test_bottleneck_configs_non_subset_intersection_and_upstream_filter(tmp_path):
+ """Verify cross-stage schema intersection config discovery (e.g. PlaidIndexBuilder -> Evaluate)
+ and downstream inclusion when filtering by an upstream-only tag."""
+ from experimaestro.tools.xpm_dag.xpm_dag import (
+ filter_subgraph_by_config,
+ get_experiment_configs,
+ stack_dag_nodes,
+ )
+
+ run_dir = tmp_path / "plaid_run"
+ run_dir.mkdir()
+ jobs_file = run_dir / "jobs.jsonl"
+
+ models = ["lightonai/LateOn", "VictorMorand/Colbert-LateOn-Pearson"]
+ docs = ["doc_a", "doc_b", "doc_c"]
+ records = []
+ full_jobs = {}
+ dependencies = {}
+ job_tags_dict = {}
+
+ PlaidCls = type("PlaidIndexBuilder", (DummyTask,), {})
+ EvalCls = type("Evaluate", (DummyTask,), {})
+
+ for m in models:
+ for d in docs:
+ idx_id = f"idx_{m}_{d}"
+ ev_id = f"eval_{m}_{d}"
+ idx_tags = {
+ "scorer": m,
+ "scorer_id": m,
+ "scorer_type": "stMultiVectorEncoder",
+ "index_documents": d,
+ }
+ ev_tags = {
+ "scorer": m,
+ "scorer_id": m,
+ "scorer_type": "stMultiVectorEncoder",
+ "dataset": f"ds_{d}",
+ "first_stage": "plaid",
+ }
+ records.append(
+ {
+ "job_id": idx_id,
+ "task_id": "xpmir.index.plaid.plaidindexbuilder",
+ "tags": idx_tags,
+ }
+ )
+ records.append(
+ {
+ "job_id": ev_id,
+ "task_id": "xpmir.evaluation.evaluate",
+ "tags": ev_tags,
+ }
+ )
+ full_jobs[idx_id] = PlaidCls()
+ full_jobs[ev_id] = EvalCls()
+ dependencies[ev_id] = [idx_id]
+ job_tags_dict[idx_id] = idx_tags
+ job_tags_dict[ev_id] = ev_tags
+
+ with jobs_file.open("w") as f:
+ for r in records:
+ f.write(json.dumps(r) + "\n")
+
+ # 1. Cross-stage intersection S_1 ∩ S_2 detects the 2 model configurations (not 12)
+ configs = get_experiment_configs(run_dir)
+ assert len(configs) == 2
+ assert configs[0] == {
+ "scorer": "lightonai/LateOn",
+ "scorer_id": "lightonai/LateOn",
+ "scorer_type": "stMultiVectorEncoder",
+ }
+ assert configs[1] == {
+ "scorer": "VictorMorand/Colbert-LateOn-Pearson",
+ "scorer_id": "VictorMorand/Colbert-LateOn-Pearson",
+ "scorer_type": "stMultiVectorEncoder",
+ }
+
+ # 2. Stacking with bottleneck grid_keys collapses 12 nodes into 4 (2 PlaidIndexBuilder x3 + 2 Evaluate x3)
+ grid_keys = set().union(*(c.keys() for c in configs))
+ s_jobs, s_deps, s_tags, s_members = stack_dag_nodes(
+ full_jobs, dependencies, job_tags_dict, grid_keys=grid_keys
+ )
+ assert len(s_jobs) == 4
+ for rep_id, members in s_members.items():
+ assert len(members) == 3
+
+ # 3. Filtering by an upstream-only tag (index_documents=doc_a) also includes downstream Evaluate
+ f_jobs, f_deps = filter_subgraph_by_config(
+ full_jobs,
+ dependencies,
+ job_tags_dict,
+ target_tags={"scorer": "lightonai/LateOn", "index_documents": "doc_a"},
+ )
+ assert set(f_jobs.keys()) == {
+ "idx_lightonai/LateOn_doc_a",
+ "eval_lightonai/LateOn_doc_a",
+ }
+ assert f_deps["eval_lightonai/LateOn_doc_a"] == ["idx_lightonai/LateOn_doc_a"]
diff --git a/src/experimaestro/tools/inspect_params/__init__.py b/src/experimaestro/tools/inspect_params/__init__.py
new file mode 100644
index 00000000..b8d3617f
--- /dev/null
+++ b/src/experimaestro/tools/inspect_params/__init__.py
@@ -0,0 +1,75 @@
+"""Experimaestro Parameters Inspector tool.
+
+Inspects params.json and experimaestro.json files with a rich, collapsible
+configuration tree, cycle detection, path resolution, and Python code synthesis.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from experimaestro.tools.inspect_params.app import (
+ NodeData,
+ NodeDetailPane,
+ NodeKind,
+ ParamsInspectorApp,
+ ParamsInspectorScreen,
+ ParamsTree,
+)
+from experimaestro.tools.inspect_params.discovery import (
+ find_local_workspace_file,
+ find_params_file,
+ find_workspace_file,
+ resolve_workspace,
+)
+from experimaestro.tools.inspect_params.models import (
+ ObjectNode,
+ ParamsGraph,
+ ResolvedPath,
+ normalize_identifier,
+ resolve_serialized_value,
+)
+from experimaestro.utils.remote import (
+ clean_job_target,
+ fetch_remote_file_rsync,
+ fetch_remote_workspace_file,
+ find_remote_workspace_file,
+ get_session_cache_dir,
+ resolve_job_file_pattern,
+ run_ssh_command,
+)
+
+__all__ = [
+ "NodeData",
+ "NodeDetailPane",
+ "NodeKind",
+ "ObjectNode",
+ "ParamsGraph",
+ "ParamsInspectorApp",
+ "ParamsInspectorScreen",
+ "ParamsTree",
+ "ResolvedPath",
+ "clean_job_target",
+ "fetch_remote_file_rsync",
+ "fetch_remote_workspace_file",
+ "find_local_workspace_file",
+ "find_params_file",
+ "find_remote_workspace_file",
+ "find_workspace_file",
+ "get_session_cache_dir",
+ "normalize_identifier",
+ "resolve_job_file_pattern",
+ "resolve_serialized_value",
+ "resolve_workspace",
+ "run_inspector",
+ "run_ssh_command",
+]
+
+
+def run_inspector(
+ target: Optional[str] = None, workspace: Optional[str] = None
+) -> None:
+ """Find parameters file and launch the interactive Textual inspector app."""
+ params_path = find_params_file(target, workspace)
+ app = ParamsInspectorApp(params_path)
+ app.run()
diff --git a/src/experimaestro/tools/inspect_params/app.py b/src/experimaestro/tools/inspect_params/app.py
new file mode 100644
index 00000000..457fb6e7
--- /dev/null
+++ b/src/experimaestro/tools/inspect_params/app.py
@@ -0,0 +1,1234 @@
+# ruff: noqa: T201
+"""Interactive Textual TUI widgets and application for Experimaestro parameters inspector."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+import json
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
+
+import pyperclip # type: ignore[import-untyped]
+from rich.console import Group
+from rich.panel import Panel
+from rich.syntax import Syntax
+from rich.table import Table
+from rich.text import Text
+
+from textual.app import App, ComposeResult
+from textual.binding import Binding
+from textual.containers import Horizontal, Vertical, VerticalScroll
+from textual.screen import Screen
+from textual import work
+from textual.widgets import Footer, Header, Input, Static, Tree
+from textual.widgets.tree import TreeNode
+
+if TYPE_CHECKING:
+ from experimaestro.scheduler.interfaces import BaseJob
+ from experimaestro.scheduler.state_provider import StateProvider
+
+from experimaestro.tools.inspect_params.models import (
+ ParamsGraph,
+ ResolvedPath,
+)
+from experimaestro.tui import APP_TCSS_PATH
+
+logger = logging.getLogger("experimaestro")
+
+
+class NodeKind(Enum):
+ """Enumeration of semantic node classifications in the parameters tree."""
+
+ ROOT_TASK = "root"
+ OBJECT = "obj"
+ CATEGORY = "cat"
+ FIELD_PRIMITIVE = "prim"
+ FIELD_PATH = "path"
+ FIELD_COLLECTION = "coll"
+ METADATA = "meta"
+ CARBON = "carbon"
+ CYCLE = "cycle"
+ SHARED_REF = "shared"
+
+
+@dataclass
+class NodeData:
+ """Payload metadata attached to a Textual Tree node.
+
+ Attributes:
+ kind: Semantic classification of the tree item.
+ title: Display title or name.
+ object_id: ID of the underlying Experimaestro object, if applicable.
+ field_name: Parameter field name if this node represents a field.
+ field_value: Raw or resolved value of the parameter.
+ resolved_path: ResolvedPath instance if this node represents a path.
+ target_object_id: Primary object ID if this node is a shared reference or jump target.
+ cycle_chain: Sequence of object IDs forming a circular reference chain.
+ is_shared: Whether this object is referenced multiple times in the DAG.
+ """
+
+ kind: NodeKind
+ title: str
+ object_id: Optional[int] = None
+ field_name: Optional[str] = None
+ field_value: Any = None
+ resolved_path: Optional[ResolvedPath] = None
+ target_object_id: Optional[int] = None
+ cycle_chain: Optional[List[int]] = None
+ is_shared: bool = False
+
+
+class ParamsTree(Tree[NodeData]):
+ """Hierarchical Task Configuration Tree widget.
+
+ Renders Experimaestro tasks, configs, hyperparameters, paths, and collections
+ in an interactive collapsible tree structure with cycle detection and shared
+ DAG reference indication.
+ """
+
+ def __init__(self, graph: ParamsGraph) -> None:
+ """Initialize the parameters tree widget.
+
+ Args:
+ graph: The indexed ParamsGraph instance to display.
+ """
+ super().__init__("Parameters")
+ self.graph = graph
+ self.group_fields: bool = True
+ self.filter_query: str = ""
+ self.show_absolute_paths: bool = False
+ self.primary_nodes: Dict[int, TreeNode[NodeData]] = {}
+ self.matched_nodes: List[TreeNode[NodeData]] = []
+ self.match_count: int = 0
+
+ def on_mount(self) -> None:
+ """Configure tree display settings and build initial tree structure."""
+ self.show_root = False
+ self.guide_depth = 3
+ self.rebuild()
+
+ def set_filter(self, q: str) -> None:
+ """Filter the tree view by search query string.
+
+ Args:
+ q: Query string to filter fields or objects.
+ """
+ self.filter_query = q.strip().lower()
+ self.rebuild()
+
+ def toggle_grouping(self) -> None:
+ """Toggle between categorized field groups and alphabetical sorting."""
+ self.group_fields = not self.group_fields
+ self.rebuild()
+
+ def toggle_paths(self) -> None:
+ """Toggle between absolute and relative path display."""
+ self.show_absolute_paths = not self.show_absolute_paths
+ self.rebuild()
+
+ def _matches(self, *texts: Any) -> bool:
+ """Check if any of the provided strings contain the active search query."""
+ if not self.filter_query:
+ return False
+ q = self.filter_query
+ return any(isinstance(t, str) and q in t.lower() for t in texts)
+
+ def _apply_highlight(self, text: Text) -> None:
+ """Highlight active search query terms in a Rich Text instance."""
+ if self.filter_query:
+ text.highlight_words(
+ [self.filter_query],
+ style="bold black on yellow",
+ case_sensitive=False,
+ )
+
+ def rebuild(self) -> None:
+ """Clear and rebuild the entire tree from the underlying graph."""
+ self.clear()
+ self.primary_nodes.clear()
+ self.matched_nodes.clear()
+ self.match_count = 0
+
+ # Metadata header
+ meta = self.root.add(
+ Text("ℹ️ Metadata & Environment", style="bold cyan"),
+ data=NodeData(NodeKind.METADATA, "Metadata"),
+ )
+ meta.add_leaf(
+ Text(f"Workspace: {self.graph.workspace_path or 'unknown'}", style="dim"),
+ data=NodeData(NodeKind.METADATA, "Workspace"),
+ )
+ meta.add_leaf(
+ Text(
+ f"v{self.graph.version} · experimaestro {self.graph.experimaestro_version}",
+ style="dim",
+ ),
+ data=NodeData(NodeKind.METADATA, "Version"),
+ )
+ if self.graph.carbon.get("enabled"):
+ meta.add_leaf(
+ Text(
+ f"🌱 Carbon: {self.graph.carbon.get('provider')} (ON)",
+ style="green",
+ ),
+ data=NodeData(NodeKind.CARBON, "Carbon", field_value=self.graph.carbon),
+ )
+
+ # Root task
+ seen: Set[int] = set()
+ rid = self.graph.root_task_id
+ if rid and rid in self.graph.objects:
+ self._render_obj(
+ self.root, rid, ancestors=(), seen=seen, label="📦 Root Task"
+ )
+ else:
+ self.root.add_leaf("[dim]No root task found[/dim]")
+
+ # Other root tasks / components if present (e.g. in multi-task graphs / experimaestro.json)
+ other_roots = [
+ oid
+ for oid, node in self.graph.objects.items()
+ if oid not in seen and not node.incoming_refs
+ ]
+ if other_roots:
+ other_container = self.root.add(
+ Text(
+ f"📋 Other Tasks & Configurations ({len(other_roots)})",
+ style="bold cyan",
+ ),
+ data=NodeData(NodeKind.CATEGORY, "Other Tasks"),
+ )
+ for roid in other_roots:
+ rnode = self.graph.objects[roid]
+ lbl = "⚡ Task" if rnode.is_task else "🧩 Config"
+ self._render_obj(
+ other_container, roid, ancestors=(), seen=seen, label=lbl
+ )
+
+ # Fallback for disconnected cycles or orphaned nodes
+ orphans = [oid for oid in self.graph.objects if oid not in seen]
+ if orphans:
+ orphan_container = self.root.add(
+ Text(
+ f"🧩 Additional Objects ({len(orphans)})",
+ style="bold yellow",
+ ),
+ data=NodeData(NodeKind.CATEGORY, "Additional Objects"),
+ )
+ for roid in orphans:
+ if roid not in seen:
+ self._render_obj(
+ orphan_container, roid, ancestors=(), seen=seen, label="Object"
+ )
+
+ if self.filter_query:
+ if self.match_count == 0:
+ self.root.add_leaf(
+ Text(
+ f"❌ No matching parameters or configs for '{self.filter_query}'",
+ style="dim red",
+ ),
+ data=NodeData(NodeKind.METADATA, "No Match"),
+ )
+ else:
+ for n in self.matched_nodes:
+ if n.children:
+ n.expand()
+ curr = n.parent
+ while curr is not None:
+ curr.expand()
+ curr = curr.parent
+ self.move_cursor(self.matched_nodes[0])
+ self.scroll_to_node(self.matched_nodes[0])
+ else:
+ for c in self.root.children:
+ c.expand()
+
+ @staticmethod
+ def _order_fields(fields: Dict[str, Any]) -> List[Tuple[str, Any]]:
+ """Order fields prioritizing core payload components over auxiliary metadata."""
+ CORE_KEYS = {
+ "model": 0,
+ "value": 0,
+ "task": 1,
+ "module": 1,
+ "dataset": 2,
+ "data": 2,
+ }
+ AUX_KEYS = {
+ "settings": 10,
+ "hooks": 11,
+ "listeners": 12,
+ "callbacks": 13,
+ }
+ return sorted(
+ fields.items(),
+ key=lambda kv: (
+ CORE_KEYS.get(kv[0], 5),
+ AUX_KEYS.get(kv[0], 5),
+ kv[0],
+ ),
+ )
+
+ def _render_obj(
+ self,
+ parent: TreeNode[NodeData],
+ oid: int,
+ ancestors: Tuple[int, ...],
+ seen: Set[int],
+ label: str = "",
+ ) -> TreeNode[NodeData]:
+ """Render an ObjectNode into the tree.
+
+ Handles cycle detection, shared DAG references, field categorization,
+ and recursive rendering of child components and init tasks.
+
+ Args:
+ parent: Parent tree node to attach to.
+ oid: Numeric ID of the object to render.
+ ancestors: Tuple of ancestor object IDs for cycle detection.
+ seen: Set of already rendered object IDs in the tree.
+ label: Optional field or role label prefix.
+
+ Returns:
+ The constructed TreeNode.
+ """
+ node = self.graph.objects[oid]
+ hid = node.identifier[:8] if node.identifier else "no-id"
+
+ # Loop / Cycle guard
+ if oid in ancestors:
+ txt = Text()
+ if label:
+ txt.append(f"{label}: ", style="bold white")
+ txt.append("🔁 [CYCLE] ", style="bold red")
+ txt.append(f"{node.type_name} [#{oid} · {hid}]", style="dim red")
+ return parent.add_leaf(
+ txt,
+ data=NodeData(
+ NodeKind.CYCLE,
+ node.type_name,
+ object_id=oid,
+ cycle_chain=list(ancestors) + [oid],
+ ),
+ )
+
+ # Shared DAG reference
+ is_shared = len(node.incoming_refs) > 1
+ if oid in seen:
+ txt = Text()
+ if label:
+ txt.append(f"{label}: ", style="bold white")
+ txt.append("↪ [Shared] ", style="bold yellow")
+ txt.append(f"{node.type_name} [#{oid} · {hid}]", style="dim yellow")
+ tn = parent.add(
+ txt,
+ data=NodeData(
+ NodeKind.SHARED_REF,
+ node.type_name,
+ object_id=oid,
+ target_object_id=oid,
+ is_shared=True,
+ ),
+ )
+ tn.add_leaf(
+ Text("(Press 'j' to jump to primary)", style="dim italic"),
+ data=NodeData(NodeKind.SHARED_REF, "Hint", target_object_id=oid),
+ )
+ return tn
+
+ seen.add(oid)
+ icon = "📦" if node.is_root_task else ("⚡" if node.is_task else "🧩")
+ st = (
+ "bold magenta"
+ if node.is_root_task
+ else ("bold cyan" if node.is_task else "bold blue")
+ )
+ txt = Text()
+ if label:
+ txt.append(f"{label}: ", style="bold white")
+ txt.append(f"{icon} {node.type_name} ", style=st)
+ txt.append(f"[{hid}] ", style="dim")
+ if is_shared:
+ txt.append(f"(shared x{len(node.incoming_refs)})", style="italic yellow")
+
+ is_match = self._matches(
+ node.type_name, node.qualified_name, node.identifier, label
+ )
+ if is_match:
+ self._apply_highlight(txt)
+
+ tree_node = parent.add(
+ txt,
+ data=NodeData(
+ NodeKind.ROOT_TASK if node.is_root_task else NodeKind.OBJECT,
+ node.qualified_name,
+ object_id=oid,
+ is_shared=is_shared,
+ ),
+ )
+ self.primary_nodes[oid] = tree_node
+ if is_match:
+ self.matched_nodes.append(tree_node)
+ self.match_count += 1
+ next_anc = ancestors + (oid,)
+
+ # Categorize fields
+ sub_c, paths, hparams, colls = [], [], [], []
+ for fn, fv in self._order_fields(node.fields):
+ if isinstance(fv, dict) and fv.get("type") == "python":
+ sub_c.append((fn, fv))
+ elif isinstance(fv, dict) and fv.get("type") in ("path", "path.serialized"):
+ paths.append((fn, fv))
+ elif isinstance(fv, (list, dict)) and not (
+ isinstance(fv, dict) and fv.get("type") == "enum"
+ ):
+ colls.append((fn, fv))
+ else:
+ hparams.append((fn, fv))
+
+ def _add_inits(target: TreeNode[NodeData]) -> None:
+ if node.init_tasks:
+ itn = target.add(
+ Text(
+ f"⚡ Init Tasks ({len(node.init_tasks)})", style="bold magenta"
+ ),
+ data=NodeData(NodeKind.CATEGORY, "Init Tasks"),
+ )
+ for i, it in enumerate(node.init_tasks):
+ if it in self.graph.objects:
+ self._render_obj(itn, it, next_anc, seen, label=f"[{i}]")
+ itn.expand()
+
+ if self.group_fields and (len(node.fields) > 3 or node.init_tasks):
+ for gtitle, gstyle, items in [
+ ("⚙️ Hyperparameters", "bold yellow", hparams),
+ ("📁 Paths & Outputs", "bold blue", paths),
+ ("🧩 Sub-Configurations", "bold cyan", sub_c),
+ ]:
+ if items:
+ gn = tree_node.add(
+ Text(gtitle, style=gstyle),
+ data=NodeData(NodeKind.CATEGORY, gtitle),
+ )
+ for fn, fv in items:
+ self._render_val(gn, fn, fv, next_anc, seen)
+ gn.expand()
+
+ for fn, fv in colls:
+ self._render_val(tree_node, fn, fv, next_anc, seen)
+ _add_inits(tree_node)
+ else:
+ for fn, fv in self._order_fields(node.fields):
+ self._render_val(tree_node, fn, fv, next_anc, seen)
+ _add_inits(tree_node)
+
+ return tree_node
+
+ def _render_val(
+ self,
+ parent: TreeNode[NodeData],
+ fn: str,
+ v: Any,
+ anc: Tuple[int, ...],
+ seen: Set[int],
+ ) -> None:
+ """Render an individual field value (sub-object, path, collection, or primitive).
+
+ Args:
+ parent: Parent tree node to attach to.
+ fn: Field name.
+ v: Field value.
+ anc: Tuple of ancestor object IDs.
+ seen: Set of visited object IDs.
+ """
+ if (
+ isinstance(v, dict)
+ and v.get("type") == "python"
+ and "value" in v
+ and int(v["value"]) in self.graph.objects
+ ):
+ self._render_obj(parent, int(v["value"]), anc, seen, label=fn)
+ return
+
+ rp = self.graph.resolve_path(v)
+ if rp:
+ is_match = self._matches(fn, str(rp.resolved), rp.raw_value)
+ txt = Text(f"{fn}: ", style="bold white")
+ pstr = (
+ str(rp.resolved)
+ if self.show_absolute_paths
+ else f"{rp.raw_value} [{rp.base}]"
+ )
+ sz = (
+ f" ({self._fmt_sz(rp.size_bytes)})"
+ if rp.exists and rp.size_bytes is not None
+ else ""
+ )
+ txt.append(
+ f"{'📁' if rp.is_dir else '📄'} {pstr}{sz} ", style="underline blue"
+ )
+ if not rp.exists:
+ txt.append("[not found]", style="red")
+ if is_match:
+ self._apply_highlight(txt)
+
+ leaf_node = parent.add_leaf(
+ txt,
+ data=NodeData(
+ NodeKind.FIELD_PATH,
+ fn,
+ field_name=fn,
+ field_value=v,
+ resolved_path=rp,
+ ),
+ )
+ if is_match:
+ self.matched_nodes.append(leaf_node)
+ self.match_count += 1
+ return
+
+ if isinstance(v, (list, dict)) and not (
+ isinstance(v, dict) and v.get("type") == "enum"
+ ):
+ is_match = self._matches(fn)
+ if isinstance(v, list):
+ desc = f"[{len(v)} items]"
+ txt = Text(f"{fn}: ", style="bold white") + Text(desc, style="cyan")
+ if is_match:
+ self._apply_highlight(txt)
+ cn = parent.add(
+ txt,
+ data=NodeData(
+ NodeKind.FIELD_COLLECTION, fn, field_name=fn, field_value=v
+ ),
+ )
+ if is_match:
+ self.matched_nodes.append(cn)
+ self.match_count += 1
+ for k_idx, val in enumerate(v):
+ self._render_val(cn, f"[{k_idx}]", val, anc, seen)
+ else:
+ desc = f"{{{len(v)} entries}}"
+ txt = Text(f"{fn}: ", style="bold white") + Text(desc, style="cyan")
+ if is_match:
+ self._apply_highlight(txt)
+ cn = parent.add(
+ txt,
+ data=NodeData(
+ NodeKind.FIELD_COLLECTION, fn, field_name=fn, field_value=v
+ ),
+ )
+ if is_match:
+ self.matched_nodes.append(cn)
+ self.match_count += 1
+ for k_key, val in sorted(v.items()):
+ self._render_val(cn, str(k_key), val, anc, seen)
+ return
+
+ is_match = self._matches(fn, str(v))
+ txt = Text(f"{fn}: ", style="bold white")
+ if isinstance(v, dict) and v.get("type") == "enum":
+ txt.append(f"{v.get('enum')}.{v.get('value')}", style="bold yellow")
+ elif isinstance(v, str):
+ txt.append(f'"{v}"', style="green")
+ elif isinstance(v, bool):
+ txt.append(str(v), style="bold yellow")
+ elif isinstance(v, (int, float)):
+ txt.append(str(v), style="magenta")
+ elif v is None:
+ txt.append("null", style="dim")
+ else:
+ txt.append(str(v), style="white")
+
+ if is_match:
+ self._apply_highlight(txt)
+
+ leaf_node = parent.add_leaf(
+ txt,
+ data=NodeData(NodeKind.FIELD_PRIMITIVE, fn, field_name=fn, field_value=v),
+ )
+ if is_match:
+ self.matched_nodes.append(leaf_node)
+ self.match_count += 1
+
+ @staticmethod
+ def _fmt_sz(sz: int) -> str:
+ """Format file size in human-readable B, KB, or MB units.
+
+ Args:
+ sz: Size in bytes.
+
+ Returns:
+ Human-readable size string.
+ """
+ if sz < 1024:
+ return f"{sz} B"
+ if sz < 1024 * 1024:
+ return f"{sz / 1024:.1f} KB"
+ return f"{sz / (1024 * 1024):.1f} MB"
+
+
+class NodeDetailPane(VerticalScroll):
+ """Right-hand detail inspector or raw JSON viewer widget.
+
+ Displays structured cards for objects, paths, primitives, carbon metrics,
+ and metadata, or syntax-highlighted raw JSON.
+ """
+
+ def __init__(self, graph: ParamsGraph) -> None:
+ """Initialize the detail inspector pane.
+
+ Args:
+ graph: The ParamsGraph instance being inspected.
+ """
+ super().__init__(id="detail-pane")
+ self.graph = graph
+ self.current_data: Optional[NodeData] = None
+ self.show_raw_json: bool = False
+
+ def compose(self) -> ComposeResult:
+ """Compose child widgets for the detail pane."""
+ yield Static(id="detail-content")
+
+ def toggle_raw_json(self) -> None:
+ """Toggle between formatted inspector card view and raw JSON view."""
+ self.show_raw_json = not self.show_raw_json
+ self.update_detail(self.current_data)
+
+ def update_detail(self, data: Optional[NodeData]) -> None:
+ """Update displayed card or raw JSON based on selected tree node data.
+
+ Args:
+ data: NodeData of the selected tree item, or None.
+ """
+ self.current_data = data
+ w = self.query_one("#detail-content", Static)
+ if not data:
+ w.update(Panel("No node selected", title="Detail Inspector"))
+ return
+
+ if self.show_raw_json:
+ target = (
+ self.graph.objects[data.object_id].fields
+ if data.object_id and data.object_id in self.graph.objects
+ else {data.field_name: data.field_value}
+ if data.field_name
+ else self.graph.raw_data
+ )
+ w.update(
+ Panel(
+ Syntax(
+ json.dumps(target, indent=2),
+ "json",
+ theme="monokai",
+ word_wrap=True,
+ ),
+ title="Raw JSON ('J' toggle)",
+ )
+ )
+ return
+
+ k = data.kind
+ if k in (
+ NodeKind.OBJECT,
+ NodeKind.ROOT_TASK,
+ NodeKind.SHARED_REF,
+ NodeKind.CYCLE,
+ ):
+ self._render_obj_card(w, data)
+ elif k == NodeKind.FIELD_PATH and data.resolved_path:
+ self._render_path_card(w, data)
+ elif k == NodeKind.FIELD_PRIMITIVE:
+ rows = [
+ ("Field", str(data.field_name)),
+ (
+ "Type",
+ type(data.field_value).__name__
+ if data.field_value is not None
+ else "null",
+ ),
+ ("Value", f"[bold yellow]{repr(data.field_value)}[/bold yellow] ('c')"),
+ ]
+ w.update(self._make_table(rows))
+ elif k == NodeKind.CARBON:
+ c = data.field_value or {}
+ rows = [
+ (
+ "Tracking",
+ "[green]ENABLED[/green]" if c.get("enabled") else "Disabled",
+ ),
+ ("Provider", str(c.get("provider"))),
+ ("Region", str(c.get("region") or "Auto")),
+ ("Interval", f"{c.get('report_interval_s')}s"),
+ ]
+ w.update(self._make_table(rows, "🌱 Carbon Tracking"))
+ elif k == NodeKind.METADATA:
+ rows = [
+ ("Workspace", str(self.graph.workspace_path or "unknown")),
+ ("Job Dir", str(self.graph.job_path or "unknown")),
+ ("Schema", f"v{self.graph.version}"),
+ ("Experimaestro", self.graph.experimaestro_version),
+ ("Objects", str(len(self.graph.objects))),
+ ]
+ w.update(self._make_table(rows, "ℹ️ Environment"))
+ else:
+ w.update(Panel(repr(data.field_value), title=data.title))
+
+ def _render_obj_card(self, w: Static, d: NodeData) -> None:
+ """Render detailed object card with properties, Python code, and references.
+
+ Args:
+ w: Target Static widget.
+ d: Selected tree node data.
+ """
+ oid = d.object_id or d.target_object_id
+ node = self.graph.objects.get(oid) if oid else None
+ if not node:
+ w.update(Panel("Unknown Object", style="red"))
+ return
+
+ role = (
+ "Root Task 📦"
+ if node.is_root_task
+ else ("Init Task ⚡" if node.is_task else "Config 🧩")
+ )
+ refs = (
+ ", ".join(f"#{c}.{f}" for c, f in node.incoming_refs)
+ if node.incoming_refs
+ else "None (Root / Unreferenced)"
+ )
+ rows = [
+ ("Type", f"[bold white]{node.type_name}[/bold white]"),
+ ("Full Name", f"[cyan]{node.qualified_name}[/cyan]"),
+ ("Object ID", f"#{node.id}"),
+ ("Role", role),
+ ("Identifier", f"[bold yellow]{node.identifier}[/bold yellow] ('c')"),
+ ("Fields", f"{len(node.fields)} fields"),
+ ]
+ parts: List[Any] = [self._make_table(rows)]
+
+ if d.kind == NodeKind.CYCLE:
+ parts.insert(
+ 0,
+ Panel(
+ f"[bold red]Circular Reference![/bold red]\nChain: {' ➔ '.join(f'#{c}' for c in (d.cycle_chain or []))}",
+ style="red",
+ title="⚠️ Loop Guard",
+ ),
+ )
+ if node.docstring:
+ parts.append(Panel(node.docstring, title="Docstring", style="dim italic"))
+
+ code = self.graph.to_python_snippet(node.id)
+ parts.append(
+ Panel(
+ Syntax(code, "python", theme="monokai", word_wrap=True),
+ title="Python .C(...) ('y' copy)",
+ subtitle="Press 'J' for Raw JSON",
+ )
+ )
+ parts.append(self._make_table([("Referenced by", refs)]))
+ w.update(Group(*parts))
+
+ def _render_path_card(self, w: Static, d: NodeData) -> None:
+ """Render detailed path card with filesystem metadata and preview.
+
+ Args:
+ w: Target Static widget.
+ d: Selected tree node data.
+ """
+ rp = d.resolved_path
+ if not rp:
+ return
+ st = "[green]✓ Exists[/green]" if rp.exists else "[red]✗ Not Found[/red]"
+ rows = [
+ ("Field", f"[bold white]{d.field_name}[/bold white]"),
+ ("Base", rp.base),
+ ("Raw Value", rp.raw_value),
+ ("Resolved", f"[cyan]{rp.resolved}[/cyan] ('c')"),
+ ("Status", st),
+ ]
+ if rp.exists:
+ rows.append(("Kind", "Directory 📁" if rp.is_dir else "File 📄"))
+ if rp.size_bytes is not None:
+ rows.append(("Size", ParamsTree._fmt_sz(rp.size_bytes)))
+ if rp.mtime:
+ rows.append(("Modified", rp.mtime.strftime("%Y-%m-%d %H:%M:%S")))
+
+ parts: List[Any] = [self._make_table(rows)]
+ if rp.exists and not rp.is_dir and (rp.size_bytes or 0) < 500_000:
+ try:
+ txt = "\n".join(
+ rp.resolved.read_text(
+ encoding="utf-8", errors="replace"
+ ).splitlines()[:12]
+ )
+ parts.append(
+ Panel(txt, title=f"Preview: {rp.resolved.name}", style="dim")
+ )
+ except Exception:
+ pass
+ w.update(Group(*parts))
+
+ @staticmethod
+ def _make_table(rows: List[Tuple[str, str]], title: Optional[str] = None) -> Any:
+ """Construct a formatted two-column table or panel.
+
+ Args:
+ rows: List of (key, value) string pairs.
+ title: Optional panel title.
+
+ Returns:
+ Rich Table or Panel instance.
+ """
+ tbl = Table(show_header=False, box=None, padding=(0, 1))
+ tbl.add_column("Key", style="bold cyan", width=14)
+ tbl.add_column("Value")
+ for k, v in rows:
+ tbl.add_row(k, v)
+ return Panel(tbl, title=title) if title else tbl
+
+
+# ---------------------------------------------------------------------------
+# Standalone App
+# ---------------------------------------------------------------------------
+
+
+# ---------------------------------------------------------------------------
+# Inspector Screen & App
+# ---------------------------------------------------------------------------
+
+
+class ParamsInspectorScreen(Screen, inherit_bindings=False):
+ """Textual Screen for inspecting experimaestro parameters.
+
+ Supports both standalone CLI usage and embedding within ExperimaestroUI.
+ Can load parameters from a local file, pre-parsed ParamsGraph, or
+ on-demand via StateProvider (for remote clusters).
+ """
+
+ CSS_PATH = APP_TCSS_PATH
+
+ BINDINGS = [
+ Binding("g", "toggle_grouping", "Group Fields", show=True),
+ Binding("p", "toggle_paths", "Paths", show=True),
+ Binding("e", "expand_all", "Expand All", show=True),
+ Binding("E", "collapse_all", "Collapse", show=True),
+ Binding("slash", "toggle_search", "Search", key_display="/", show=True),
+ Binding("c", "copy_value", "Copy", show=True),
+ Binding("y", "copy_python", "Copy Python", show=True),
+ Binding("J", "toggle_raw_json", "Raw JSON", key_display="J", show=True),
+ Binding("j", "jump_primary", "Jump", show=True),
+ Binding("escape", "dismiss_or_quit", "Back", show=True, priority=True),
+ Binding("q", "dismiss_or_quit", "Back", show=False, priority=True),
+ ]
+
+ def __init__(
+ self,
+ params_target: Path | str | ParamsGraph | None = None,
+ job: Optional["BaseJob"] = None,
+ state_provider: Optional["StateProvider"] = None,
+ title: Optional[str] = None,
+ ):
+ """Initialize the parameters inspector screen.
+
+ Args:
+ params_target: Filesystem path or already-parsed ParamsGraph.
+ job: BaseJob instance for StateProvider retrieval.
+ state_provider: StateProvider for fetching parameters.
+ title: Custom title for the screen header.
+ """
+ super().__init__()
+ self.job = job
+ self.state_provider = state_provider
+ self.custom_title = title
+ self.params_path: Optional[Path] = None
+ self.graph: Optional[ParamsGraph] = None
+ self._loading = False
+
+ if isinstance(params_target, ParamsGraph):
+ self.graph = params_target
+ if self.graph.job_path:
+ self.params_path = self.graph.job_path
+ elif params_target is not None:
+ self.params_path = Path(params_target).resolve()
+ self.graph = ParamsGraph.from_file(self.params_path)
+ elif self.job is not None and self.state_provider is not None:
+ if not self.state_provider.is_remote:
+ p = self.state_provider.get_job_params_file(self.job)
+ if p:
+ self.params_path = p
+ self.graph = ParamsGraph.from_file(p)
+ else:
+ self._loading = True
+ else:
+ self._loading = True
+ else:
+ raise ValueError(
+ "Either params_target or (job and state_provider) must be provided"
+ )
+
+ if self.custom_title:
+ self.title = self.custom_title
+ elif self.params_path and self.graph:
+ jname = self.graph.job_path.name if self.graph.job_path else "job"
+ self.sub_title = f"{jname} ({self.params_path.parent.parent.name if self.graph.job_path else ''})"
+
+ def compose(self) -> ComposeResult:
+ """Compose top-level layout of the screen."""
+ yield Header()
+ if self._loading:
+ yield Static(
+ "Syncing parameters from remote...",
+ id="loading",
+ classes="loading-message",
+ )
+ else:
+ with Horizontal(id="mode-bar"):
+ yield Static(" ⚙️ Config Tree ", id="view-title", classes="title-badge")
+ yield Static(
+ " [g] Group · [p] Paths · [J] Raw JSON · [/] Search",
+ id="mode-hints",
+ )
+ with Horizontal(id="search-bar-container", classes="hidden"):
+ yield Input(placeholder="Filter parameters...", id="search-input")
+ yield Static("", id="search-status")
+ with Horizontal(id="split-container"):
+ with Vertical(id="tree-container"):
+ yield ParamsTree(self.graph)
+ yield NodeDetailPane(self.graph)
+ yield Footer()
+
+ def on_mount(self) -> None:
+ """Set initial focus or start remote sync on mount."""
+ if self._loading:
+ self._start_remote_sync()
+ else:
+ self.query_one(ParamsTree).focus()
+
+ @work(thread=True)
+ def _start_remote_sync(self) -> None:
+ """Fetch parameters file via StateProvider in background thread."""
+ try:
+ if self.state_provider and self.job:
+ p = self.state_provider.get_job_params_file(self.job)
+ if p and p.is_file():
+ self.params_path = p
+ self.graph = ParamsGraph.from_file(p)
+ self.app.call_from_thread(self._on_sync_success)
+ return
+ self.app.call_from_thread(
+ self._on_sync_failure, "Parameters file not found"
+ )
+ except Exception as e:
+ self.app.call_from_thread(self._on_sync_failure, str(e))
+
+ def _on_sync_success(self) -> None:
+ """Mount inspector widgets once remote parameters have been retrieved."""
+ self._loading = False
+ try:
+ loading = self.query_one("#loading", Static)
+ loading.remove()
+ except Exception:
+ pass
+
+ footer = self.query_one(Footer)
+ mode_bar = Horizontal(
+ Static(" ⚙️ Config Tree ", id="view-title", classes="title-badge"),
+ Static(
+ " [g] Group · [p] Paths · [J] Raw JSON · [/] Search",
+ id="mode-hints",
+ ),
+ id="mode-bar",
+ )
+ search_bar = Horizontal(
+ Input(placeholder="Filter parameters...", id="search-input"),
+ Static("", id="search-status"),
+ id="search-bar-container",
+ classes="hidden",
+ )
+ split = Horizontal(
+ Vertical(ParamsTree(self.graph), id="tree-container"),
+ NodeDetailPane(self.graph),
+ id="split-container",
+ )
+ self.mount(mode_bar, before=footer)
+ self.mount(search_bar, before=footer)
+ self.mount(split, before=footer)
+
+ if not self.custom_title and self.params_path and self.graph:
+ jname = self.graph.job_path.name if self.graph.job_path else "job"
+ self.sub_title = f"{jname} ({self.params_path.parent.parent.name if self.graph.job_path else ''})"
+
+ tree = self.query_one(ParamsTree)
+ tree.focus()
+
+ def _on_sync_failure(self, error_msg: str) -> None:
+ """Display error message when sync fails."""
+ try:
+ loading = self.query_one("#loading", Static)
+ loading.update(f"⚠️ {error_msg}\n\nPress Esc or 'q' to go back.")
+ except Exception:
+ pass
+
+ def on_tree_node_highlighted(self, event: Tree.NodeHighlighted) -> None:
+ """Update detail inspector pane when cursor moves over a node."""
+ self.query_one(NodeDetailPane).update_detail(event.node.data)
+
+ def on_tree_node_selected(self, event: Tree.NodeSelected) -> None:
+ """Activate selected node, jumping to primary definition if shared."""
+ if event.node.data and event.node.data.target_object_id:
+ self.action_jump_primary()
+
+ def on_input_changed(self, event: Input.Changed) -> None:
+ """Filter tree view as query string changes."""
+ tree = self.query_one(ParamsTree)
+ tree.set_filter(event.value)
+ inp = self.query_one("#search-input", Input)
+ status = self.query_one("#search-status", Static)
+ hints = self.query_one("#mode-hints", Static)
+
+ q = event.value.strip()
+ if q:
+ if tree.match_count == 0:
+ inp.add_class("error")
+ inp.remove_class("valid")
+ status.remove_class("valid")
+ status.add_class("error")
+ status.update(" (not found)")
+ hints.update(
+ f" [g] Group · [p] Paths · [/] Search: '{q}' [bold red](not found)[/bold red]"
+ )
+ else:
+ inp.remove_class("error")
+ inp.add_class("valid")
+ status.remove_class("error")
+ status.add_class("valid")
+ status.update(f" ({tree.match_count} matched)")
+ hints.update(
+ f" [g] Group · [p] Paths · [/] Search: '{q}' [green]({tree.match_count} matched)[/green]"
+ )
+ else:
+ inp.remove_class("error")
+ inp.remove_class("valid")
+ status.remove_class("error")
+ status.remove_class("valid")
+ status.update("")
+ hints.update(" [g] Group · [p] Paths · [J] Raw JSON · [/] Search")
+
+ def on_input_submitted(self, event: Input.Submitted) -> None:
+ """Hide search bar and return focus to tree upon query submission."""
+ self.query_one("#search-bar-container").add_class("hidden")
+ self.query_one(ParamsTree).focus()
+
+ def action_toggle_grouping(self) -> None:
+ """Toggle categorized field groups versus flat alphabetical ordering."""
+ tree = self.query_one(ParamsTree)
+ tree.toggle_grouping()
+ self.notify(
+ f"Field grouping: {'Categorized' if tree.group_fields else 'Alphabetical'}"
+ )
+
+ def action_toggle_paths(self) -> None:
+ """Toggle absolute filesystem path versus raw relative path display."""
+ tree = self.query_one(ParamsTree)
+ tree.toggle_paths()
+ self.notify(
+ f"Path display: {'Absolute' if tree.show_absolute_paths else 'Relative'}"
+ )
+
+ def action_toggle_raw_json(self) -> None:
+ """Toggle raw JSON view versus structured inspector card in detail pane."""
+ pane = self.query_one(NodeDetailPane)
+ pane.toggle_raw_json()
+ self.notify(
+ f"Detail pane: {'Raw JSON' if pane.show_raw_json else 'Inspector Card'}"
+ )
+
+ def action_expand_all(self) -> None:
+ """Recursively expand all nodes in the parameter tree."""
+
+ def _exp(n: TreeNode[NodeData]) -> None:
+ n.expand()
+ for c in n.children:
+ _exp(c)
+
+ _exp(self.query_one(ParamsTree).root)
+ self.notify("Expanded all")
+
+ def action_collapse_all(self) -> None:
+ """Collapse all child nodes in the parameter tree."""
+
+ def _col(n: TreeNode[NodeData]) -> None:
+ for c in n.children:
+ _col(c)
+ c.collapse()
+
+ _col(self.query_one(ParamsTree).root)
+ self.notify("Collapsed all")
+
+ def action_toggle_search(self) -> None:
+ """Toggle visibility and focus of the search filter input."""
+ box = self.query_one("#search-bar-container")
+ if "hidden" in box.classes:
+ box.remove_class("hidden")
+ self.query_one("#search-input", Input).focus()
+ else:
+ box.add_class("hidden")
+ self.query_one(ParamsTree).focus()
+
+ def action_copy_value(self) -> None:
+ """Copy the identifier, resolved path, or raw value of the selected node."""
+ node = self.query_one(ParamsTree).cursor_node
+ if not node or not node.data:
+ return
+ d = node.data
+ val = (
+ self.graph.objects[d.object_id].identifier
+ if d.object_id
+ and d.kind in (NodeKind.OBJECT, NodeKind.ROOT_TASK, NodeKind.SHARED_REF)
+ else str(d.resolved_path.resolved)
+ if d.resolved_path
+ else str(d.field_value)
+ if d.field_value is not None
+ else ""
+ )
+ if val:
+ pyperclip.copy(val)
+ self.notify(f"Copied: {val[:32]}...")
+
+ def action_copy_python(self) -> None:
+ """Copy the reproducible Python constructor snippet of the selected object."""
+ node = self.query_one(ParamsTree).cursor_node
+ oid = node.data.object_id if node and node.data else self.graph.root_task_id
+ if oid and oid in self.graph.objects:
+ pyperclip.copy(self.graph.to_python_snippet(oid))
+ self.notify("Copied Python .C(...) constructor!")
+
+ def action_jump_primary(self) -> None:
+ """Navigate to the primary definition node in the tree from a shared reference."""
+ node = self.query_one(ParamsTree).cursor_node
+ if not node or not node.data:
+ return
+ tid = node.data.target_object_id or node.data.object_id
+ tree = self.query_one(ParamsTree)
+ if tid and tid in tree.primary_nodes:
+ target = tree.primary_nodes[tid]
+
+ # 1. Expand all ancestors from root down to target so it is visible
+ curr = target.parent
+ while curr is not None:
+ if not curr.is_expanded:
+ curr.expand()
+ curr = curr.parent
+
+ # 2. Expand target itself so its parameters/fields are revealed
+ target.expand()
+ for child in target.children:
+ if child.data and child.data.kind == NodeKind.CATEGORY:
+ child.expand()
+
+ # 3. Move cursor to target and scroll into view
+ tree.move_cursor(target)
+ tree.scroll_to_node(target)
+ self.query_one(NodeDetailPane).update_detail(target.data)
+ self.notify(f"Jumped to primary definition #{tid}")
+
+ def action_dismiss_or_quit(self) -> None:
+ """Dismiss the search bar if visible, or exit the application/screen."""
+ if not self._loading:
+ try:
+ box = self.query_one("#search-bar-container")
+ if "hidden" not in box.classes:
+ box.add_class("hidden")
+ inp = self.query_one("#search-input", Input)
+ inp.value = ""
+ inp.remove_class("error")
+ inp.remove_class("valid")
+ status = self.query_one("#search-status", Static)
+ status.remove_class("error")
+ status.remove_class("valid")
+ status.update("")
+ self.query_one("#mode-hints", Static).update(
+ " [g] Group · [p] Paths · [J] Raw JSON · [/] Search"
+ )
+ tree = self.query_one(ParamsTree)
+ tree.set_filter("")
+ tree.focus()
+ return
+ except Exception:
+ pass
+
+ if len(self.app.screen_stack) > 1:
+ self.dismiss()
+ else:
+ self.app.exit()
+
+
+class ParamsInspectorApp(App):
+ """Standalone Textual App for inspecting experimaestro params.json.
+
+ Features interactive tree navigation, categorization, path resolution,
+ fuzzy search filtering, Python snippet copying, and raw JSON inspection.
+ """
+
+ TITLE = "Experimaestro Parameters Inspector"
+ CSS_PATH = APP_TCSS_PATH
+
+ def __init__(self, params_path: Path):
+ """Initialize the parameters inspector application.
+
+ Args:
+ params_path: Filesystem path to the parameters JSON file.
+ """
+ super().__init__()
+ self.params_path = Path(params_path).resolve()
+ self.inspector_screen = ParamsInspectorScreen(self.params_path)
+
+ def get_default_screen(self) -> Screen:
+ """Return the inspector screen as the default screen."""
+ return self.inspector_screen
+
+ @property
+ def graph(self) -> Optional[ParamsGraph]:
+ """Accessor for the underlying ParamsGraph."""
+ return self.inspector_screen.graph
+
+ def action_expand_all(self) -> None:
+ """Forward expand_all to inspector screen."""
+ self.inspector_screen.action_expand_all()
+
+ def action_collapse_all(self) -> None:
+ """Forward collapse_all to inspector screen."""
+ self.inspector_screen.action_collapse_all()
+
+ def action_toggle_grouping(self) -> None:
+ """Forward toggle_grouping to inspector screen."""
+ self.inspector_screen.action_toggle_grouping()
+
+ def action_toggle_paths(self) -> None:
+ """Forward toggle_paths to inspector screen."""
+ self.inspector_screen.action_toggle_paths()
+
+ def action_toggle_raw_json(self) -> None:
+ """Forward toggle_raw_json to inspector screen."""
+ self.inspector_screen.action_toggle_raw_json()
+
+ def action_toggle_search(self) -> None:
+ """Forward toggle_search to inspector screen."""
+ self.inspector_screen.action_toggle_search()
+
+ def action_copy_value(self) -> None:
+ """Forward copy_value to inspector screen."""
+ self.inspector_screen.action_copy_value()
+
+ def action_copy_python(self) -> None:
+ """Forward copy_python to inspector screen."""
+ self.inspector_screen.action_copy_python()
+
+ def action_jump_primary(self) -> None:
+ """Forward jump_primary to inspector screen."""
+ self.inspector_screen.action_jump_primary()
diff --git a/src/experimaestro/tools/inspect_params/discovery.py b/src/experimaestro/tools/inspect_params/discovery.py
new file mode 100644
index 00000000..92ff67ed
--- /dev/null
+++ b/src/experimaestro/tools/inspect_params/discovery.py
@@ -0,0 +1,261 @@
+# ruff: noqa: T201
+"""File and workspace discovery utilities for Experimaestro parameters inspector.
+
+Resolves local workspace paths, remote SSH workspaces (via `experimaestro.utils.remote`),
+and locates params.json / experimaestro.json files.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+import sys
+from typing import Any, Optional, Union
+
+from experimaestro.utils.remote import (
+ clean_job_target,
+ find_remote_workspace_file,
+ resolve_job_file_pattern,
+)
+
+logger = logging.getLogger("experimaestro")
+
+
+def resolve_workspace(
+ workspace: Optional[str] = None,
+) -> Union[Path, Any]:
+ """Resolve workspace directory from an explicit name/path or Experimaestro default.
+
+ Resolution order:
+ 1. If workspace argument is provided:
+ - If it points to an existing local directory, return it.
+ - Otherwise, look up configured workspace settings via `find_workspace(workspace=...)`.
+ 2. If no workspace is provided:
+ - Look up default workspace via `find_workspace()`.
+ - Check `EXPERIMAESTRO_WORKSPACE` environment variable.
+ - Search parent directories for `.experimaestro`.
+
+ Args:
+ workspace: Optional workspace name or filesystem path.
+
+ Returns:
+ Resolved Path to the local workspace directory, or remote WorkspaceSettings.
+
+ Raises:
+ FileNotFoundError: If the workspace cannot be determined or does not exist.
+ """
+ if workspace:
+ p = Path(workspace).expanduser().resolve()
+ if p.is_dir():
+ logger.info("Using workspace %s", p)
+ return p
+ try:
+ from experimaestro.settings import find_workspace
+
+ ws_env = find_workspace(workspace=workspace)
+ if ws_env:
+ if ws_env.is_remote:
+ return ws_env
+ if ws_env.path:
+ return Path(ws_env.path).resolve()
+ except Exception:
+ pass
+ raise FileNotFoundError(
+ f"Specified workspace '{workspace}' could not be found."
+ )
+
+ # Try default workspace from experimaestro settings
+ try:
+ from experimaestro.settings import find_workspace
+
+ ws_env = find_workspace()
+ if ws_env:
+ if ws_env.is_remote:
+ return ws_env
+ if ws_env.path:
+ return Path(ws_env.path).resolve()
+ except Exception:
+ pass
+
+ # Fallback: check environment variable or walk up from current working directory
+ env_ws = os.environ.get("EXPERIMAESTRO_WORKSPACE")
+ if env_ws and Path(env_ws).is_dir():
+ ws_path = Path(env_ws).resolve()
+ logger.info("Using default workspace %s", ws_path)
+ return ws_path
+
+ for p in [Path.cwd()] + list(Path.cwd().parents):
+ if (p / ".experimaestro").is_dir():
+ ws_path = p.resolve()
+ logger.info("Using default workspace %s", ws_path)
+ return ws_path
+
+ raise FileNotFoundError(
+ "No workspace specified and default workspace could not be determined."
+ )
+
+
+def find_local_workspace_file(
+ ws_dir: Path, target: str, filename: str = "params.json"
+) -> Path:
+ """Find a specific file within a local workspace job directory.
+
+ Args:
+ ws_dir: Local workspace root directory.
+ target: Target job identifier, prefix, or job path.
+ filename: Relative filename or pattern to locate (default: 'params.json').
+
+ Returns:
+ Resolved Path to the matching file.
+
+ Raises:
+ FileNotFoundError: If target is empty or no matching file exists in workspace.
+ """
+ try:
+ clean = clean_job_target(target, str(ws_dir), filename=filename)
+ except ValueError as e:
+ raise FileNotFoundError(
+ f"Please specify a job ID or path. {filename} not found in current directory."
+ ) from e
+
+ pattern = resolve_job_file_pattern(clean, filename)
+ matches = sorted(list(ws_dir.glob(pattern)))
+
+ if not matches:
+ raise FileNotFoundError(
+ f"No job matching '{clean}' found in workspace {ws_dir} (pattern: {pattern})"
+ )
+
+ if len(matches) > 1:
+ print(
+ f"Warning: Multiple jobs ({len(matches)}) matched '{clean}'. "
+ f"Using the first match: {matches[0].parent.name} ({matches[0].parent.parent.name})",
+ file=sys.stderr,
+ )
+
+ return matches[0].resolve()
+
+
+def find_workspace_file(
+ ws_or_path: Any,
+ target: str,
+ filename: str = "params.json",
+ cache_dir: Optional[Path] = None,
+) -> Path:
+ """Find or fetch a file from either a local or remote Experimaestro workspace.
+
+ Args:
+ ws_or_path: WorkspaceSettings instance (remote/local) or Path to local workspace.
+ target: Job ID prefix, locator, or job directory path.
+ filename: Relative filename within the job directory (default: 'params.json').
+ cache_dir: Optional local cache directory when fetching from a remote workspace.
+
+ Returns:
+ Resolved local Path to the found or synchronized file.
+ """
+ from experimaestro.settings import WorkspaceSettings
+
+ if isinstance(ws_or_path, WorkspaceSettings) and ws_or_path.is_remote:
+ return find_remote_workspace_file(
+ ws_or_path, target, filename=filename, cache_dir=cache_dir
+ )
+
+ ws_dir = (
+ ws_or_path.path
+ if isinstance(ws_or_path, WorkspaceSettings)
+ else Path(ws_or_path)
+ )
+ return find_local_workspace_file(ws_dir, target, filename=filename)
+
+
+def _find_params_in_path(p: Path) -> Optional[Path]:
+ """Check if a path points directly to, or contains, a parameters JSON file.
+
+ Checks if `p` is a file directly, or if it is a directory containing
+ `params.json` or `experimaestro.json` (or any descendant match).
+
+ Args:
+ p: Path to inspect.
+
+ Returns:
+ Resolved Path to the found file, or None if no match is found.
+ """
+ if p.is_file():
+ return p.resolve()
+ if p.is_dir():
+ for name in ("params.json", "experimaestro.json", "objects.jsonl"):
+ cand = p / name
+ if cand.is_file():
+ return cand.resolve()
+ m = (
+ list(p.glob("**/params.json"))
+ + list(p.glob("**/experimaestro.json"))
+ + list(p.glob("**/objects.jsonl"))
+ )
+ if m:
+ return m[0].resolve()
+ return None
+
+
+def find_params_file(
+ target: Optional[str] = None, workspace: Optional[str] = None
+) -> Path:
+ """Find and resolve the path to a params.json or experimaestro.json file.
+
+ Search resolution order:
+ 1. If an explicit workspace is specified:
+ - If remote, query and synchronize via SSH.
+ - If target is an existing local absolute path, return it directly.
+ - Otherwise, search inside the specified local workspace.
+ 2. If no workspace is specified:
+ - If target is an absolute or user-tilde path, resolve locally; do NOT
+ attempt workspace discovery.
+ - If target is an existing local relative path/directory, resolve it.
+ - Otherwise, discover the default workspace (remote or local) and search
+ for matching jobs.
+
+ Args:
+ target: Job ID / hash prefix, path to params.json, or job directory.
+ workspace: Optional explicit workspace name or path.
+
+ Returns:
+ Resolved local Path to the parameters JSON file.
+
+ Raises:
+ FileNotFoundError: If the target cannot be found or matched.
+ """
+ raw = target or "."
+ p = Path(raw).expanduser()
+
+ # 1. If explicit workspace is specified, check it first (remote overrides local path)
+ if workspace is not None:
+ ws_or_path = resolve_workspace(workspace)
+ from experimaestro.settings import WorkspaceSettings
+
+ if isinstance(ws_or_path, WorkspaceSettings) and ws_or_path.is_remote:
+ return find_workspace_file(ws_or_path, raw, filename="params.json")
+
+ # Explicit local workspace: if target is an existing local absolute path, return it directly
+ if p.is_absolute():
+ resolved = _find_params_in_path(p)
+ if resolved:
+ return resolved
+
+ return find_workspace_file(ws_or_path, raw, filename="params.json")
+
+ # 2. No explicit workspace: check if absolute or tilde path exists locally (never load from workspace)
+ if p.is_absolute():
+ resolved = _find_params_in_path(p)
+ if resolved:
+ return resolved
+ raise FileNotFoundError(f"params.json not found at absolute path: {raw}")
+
+ # 3. Existing relative path locally
+ resolved = _find_params_in_path(p)
+ if resolved:
+ return resolved
+
+ # 4. Otherwise fall back to default workspace discovery
+ ws_or_path = resolve_workspace()
+ return find_workspace_file(ws_or_path, raw, filename="params.json")
diff --git a/src/experimaestro/tools/inspect_params/models.py b/src/experimaestro/tools/inspect_params/models.py
new file mode 100644
index 00000000..524ee4d2
--- /dev/null
+++ b/src/experimaestro/tools/inspect_params/models.py
@@ -0,0 +1,406 @@
+"""Core data models and dependency graph for Experimaestro parameters inspector.
+
+Represents serialized objects, handles reference traversal, cycle detection,
+path resolution, and Python code synthesis.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+import importlib
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union
+
+from experimaestro.core.identifier import Identifier
+
+logger = logging.getLogger("experimaestro")
+
+
+def normalize_identifier(raw_ident: Any) -> str:
+ """Normalize a serialized identifier (string hash or Identifier state dict)."""
+ if isinstance(raw_ident, dict):
+ return str(Identifier.from_state_dict(raw_ident))
+ return str(raw_ident) if raw_ident is not None else ""
+
+
+def resolve_serialized_value(value: Any, objects: Mapping[int, Any]) -> Any:
+ """Resolve a serialized JSON field value against a mapping of loaded objects."""
+ if isinstance(value, list):
+ return [resolve_serialized_value(x, objects) for x in value]
+
+ if isinstance(value, dict):
+ if "type" not in value:
+ return {k: resolve_serialized_value(v, objects) for k, v in value.items()}
+
+ vtype = value["type"]
+ if vtype == "python":
+ ref_id = int(value["value"])
+ if ref_id not in objects:
+ logger.warning(
+ "Referenced object id %s not found in loaded objects", ref_id
+ )
+ return None
+ return objects[ref_id]
+
+ if vtype in ("path", "path.serialized"):
+ return Path(value["value"])
+
+ if vtype == "set":
+ return [
+ resolve_serialized_value(x, objects) for x in value.get("items", [])
+ ]
+
+ if vtype == "enum":
+ return value.get("value")
+
+ return value.get("value")
+
+ return value
+
+
+@dataclass
+class ResolvedPath:
+ """Representation of a path reference resolved against job or workspace directories.
+
+ Attributes:
+ raw_value: Original string representation from the parameters JSON.
+ base: Base reference origin ('job', 'workspace', or 'absolute').
+ resolved: Fully resolved filesystem Path.
+ exists: Whether the file or directory exists on the local filesystem.
+ is_dir: Whether the resolved path is a directory.
+ size_bytes: File size in bytes if it exists and is a regular file.
+ mtime: Last modification timestamp if accessible.
+ """
+
+ raw_value: str
+ base: str # 'job' | 'workspace' | 'absolute'
+ resolved: Path
+ exists: bool = False
+ is_dir: bool = False
+ size_bytes: Optional[int] = None
+ mtime: Optional[datetime] = None
+
+ @classmethod
+ def resolve(
+ cls, raw_val: str, base: str, job_dir: Optional[Path], ws_dir: Optional[Path]
+ ) -> ResolvedPath:
+ """Resolve a raw path string relative to job or workspace directory.
+
+ Resolution precedence:
+ 1. If base is 'job' and job_dir is provided: resolve relative to job_dir.
+ 2. If base is 'workspace' and ws_dir is provided: resolve relative to ws_dir.
+ 3. If raw path is relative and job_dir is provided: resolve relative to job_dir.
+ 4. Otherwise, resolve raw path directly.
+ """
+ p = Path(raw_val)
+ res = (
+ (job_dir / p).resolve()
+ if base == "job" and job_dir
+ else (ws_dir / p).resolve()
+ if base == "workspace" and ws_dir
+ else (job_dir / p).resolve()
+ if not p.is_absolute() and job_dir
+ else p.resolve()
+ )
+ exists = res.exists()
+ is_dir = res.is_dir() if exists else False
+ size, mtime = None, None
+ if exists and not is_dir:
+ try:
+ st = res.stat()
+ size, mtime = st.st_size, datetime.fromtimestamp(st.st_mtime)
+ except OSError:
+ pass
+ return cls(raw_val, base, res, exists, is_dir, size, mtime)
+
+
+@dataclass
+class ObjectNode:
+ """A deserialized configuration object or task within the Experimaestro graph.
+
+ Attributes:
+ id: Unique numeric object identifier.
+ module: Python module declaring the object's class.
+ type_name: Short class name of the object.
+ qualified_name: Full module.class or qualified typename.
+ identifier: Content hash or task identifier string.
+ is_task: True if this object represents an executable task.
+ is_root_task: True if this is the primary root task of the job.
+ init_tasks: IDs of prerequisite tasks that must run before this task.
+ fields: Key-value mapping of parameters, sub-configs, or paths.
+ incoming_refs: List of (source_object_id, field_name) referencing this object.
+ outgoing_refs: List of (field_name, target_object_id) referenced by this object.
+ docstring: Optional docstring extracted from the underlying Python class.
+ """
+
+ id: int
+ module: str
+ type_name: str
+ qualified_name: str
+ identifier: str
+ is_task: bool
+ is_root_task: bool
+ init_tasks: List[int]
+ fields: Dict[str, Any]
+ incoming_refs: List[Tuple[int, str]] = field(default_factory=list)
+ outgoing_refs: List[Tuple[str, int]] = field(default_factory=list)
+ docstring: Optional[str] = None
+ task_id: Optional[int] = None
+ is_init_task: bool = False
+
+ @classmethod
+ def from_definition(
+ cls, o: Dict[str, Any], *, load_docstring: bool = True
+ ) -> ObjectNode:
+ """Create an ObjectNode from a raw serialized definition dictionary."""
+ oid = int(o["id"])
+ mod, typ = o.get("module", ""), o.get("type", "")
+ qname = o.get("typename") or f"{mod}.{typ}" if mod else typ or str(oid)
+ ident = normalize_identifier(o.get("identifier", ""))
+ inits = [int(x) for x in o.get("init-tasks", [])]
+ t_ptr = o.get("task")
+ task_id = int(t_ptr) if t_ptr is not None else None
+ is_task = (task_id is not None and task_id == oid) or bool(inits)
+
+ doc = None
+ if load_docstring and mod and typ:
+ try:
+ cls_obj = getattr(importlib.import_module(mod), typ, None)
+ doc = cls_obj.__doc__.strip() if cls_obj and cls_obj.__doc__ else None
+ except Exception:
+ pass
+
+ raw_fields = o.get("fields", {})
+ return cls(
+ oid,
+ mod,
+ typ,
+ qname,
+ ident,
+ is_task,
+ False,
+ inits,
+ raw_fields if isinstance(raw_fields, dict) else {},
+ docstring=doc,
+ task_id=task_id,
+ )
+
+
+class ParamsGraph:
+ """Parses, indexes, and queries the object dependency graph of an Experimaestro job.
+
+ Supports single-job dictionaries (`params.json`), multi-task list payloads
+ (such as Hugging Face Hub `experimaestro.json` exports), and streaming
+ experiment records (`objects.jsonl`).
+
+ Attributes:
+ raw_data: Original deserialized JSON data.
+ job_path: Path to the directory containing the parameters file.
+ workspace_path: Resolved path to the enclosing workspace directory.
+ objects: Mapping of numeric object IDs to ObjectNode instances.
+ task_object_ids: Set of object IDs referenced via `task` or `init-tasks`.
+ root_task_id: Object ID of the primary root task, if determined.
+ version: Schema version number.
+ experimaestro_version: Experimaestro version string.
+ carbon: Carbon emission tracking configuration dictionary.
+ """
+
+ workspace_path: Optional[Path]
+
+ def __init__(
+ self,
+ data: Union[Dict[str, Any], List[Dict[str, Any]]],
+ job_path: Optional[Path] = None,
+ workspace_path: Optional[Path] = None,
+ *,
+ load_docstrings: bool = True,
+ ):
+ """Initialize the graph from a dictionary or list of raw object specifications.
+
+ Args:
+ data: Deserialized params.json dict or experimaestro.json / objects.jsonl list.
+ job_path: Optional filesystem path to the enclosing job directory.
+ workspace_path: Optional workspace path (auto-discovered if omitted).
+ load_docstrings: Whether to import modules to extract class docstrings.
+ """
+ self.raw_data = data
+ self.job_path = job_path.resolve() if job_path else None
+ if isinstance(data, list):
+ ws = None
+ self.version = 0
+ self.experimaestro_version = "unknown"
+ self.carbon = {}
+ raw_objs = data
+ else:
+ ws = data.get("workspace")
+ self.version = data.get("version", 0)
+ self.experimaestro_version = data.get("experimaestro", "unknown")
+ self.carbon = data.get("carbon", {})
+ raw_objs = data.get("objects", [])
+
+ self.workspace_path = (
+ Path(ws).resolve()
+ if ws
+ else workspace_path.resolve()
+ if workspace_path
+ else self._find_ws(self.job_path)
+ )
+ self.objects: Dict[int, ObjectNode] = {}
+ self.task_object_ids: Set[int] = set()
+ self.init_task_object_ids: Set[int] = set()
+ self.root_task_id: Optional[int] = None
+ self._parse(raw_objs, load_docstrings=load_docstrings)
+
+ @staticmethod
+ def _find_ws(path: Optional[Path]) -> Optional[Path]:
+ """Search upwards from a directory for the nearest `.experimaestro` root."""
+ if not path:
+ return None
+ for p in [path] + list(path.parents):
+ if (p / ".experimaestro").is_dir():
+ return p
+ return None
+
+ @classmethod
+ def from_file(cls, path: Path, *, load_docstrings: bool = True) -> ParamsGraph:
+ """Load a parameter graph from a file or directory.
+
+ Accepts direct paths to `params.json`, `experimaestro.json`, or `objects.jsonl`,
+ or a directory containing any of these files.
+ """
+ p = path.resolve()
+ if p.is_dir():
+ if (p / "params.json").is_file():
+ f = p / "params.json"
+ elif (p / "experimaestro.json").is_file():
+ f = p / "experimaestro.json"
+ elif (p / "objects.jsonl").is_file():
+ f = p / "objects.jsonl"
+ else:
+ f = p / "params.json"
+ else:
+ f = p
+ if not f.exists():
+ raise FileNotFoundError(f"params.json not found at: {f}")
+
+ text = f.read_text(encoding="utf-8")
+ if f.suffix == ".jsonl" or f.name == "objects.jsonl":
+ payload: Union[Dict[str, Any], List[Dict[str, Any]]] = [
+ json.loads(line) for line in text.splitlines() if line.strip()
+ ]
+ else:
+ payload = json.loads(text)
+ return cls(payload, job_path=f.parent, load_docstrings=load_docstrings)
+
+ def _parse(
+ self, raw_objs: List[Dict[str, Any]], *, load_docstrings: bool = True
+ ) -> None:
+ """Construct ObjectNode instances and resolve reference graphs."""
+ target_hash = self.job_path.name if self.job_path else None
+ cand_root = None
+
+ for o in raw_objs:
+ node = ObjectNode.from_definition(o, load_docstring=load_docstrings)
+ self.objects[node.id] = node
+ if node.task_id is not None:
+ self.task_object_ids.add(node.task_id)
+ for it in node.init_tasks:
+ self.init_task_object_ids.add(it)
+ if target_hash and node.identifier == target_hash:
+ cand_root = node.id
+
+ for tid in self.task_object_ids:
+ if tid in self.objects:
+ self.objects[tid].is_task = True
+
+ for it_id in self.init_task_object_ids:
+ if it_id in self.objects and not self.objects[it_id].is_task:
+ self.objects[it_id].is_init_task = True
+
+ self.root_task_id = (
+ cand_root if cand_root else (int(raw_objs[-1]["id"]) if raw_objs else None)
+ )
+ if self.root_task_id and self.root_task_id in self.objects:
+ self.objects[self.root_task_id].is_root_task = True
+ self.objects[self.root_task_id].is_task = True
+
+ def _extract_refs(val: Any, prefix: str = "") -> List[Tuple[str, int]]:
+ refs: List[Tuple[str, int]] = []
+ if isinstance(val, dict):
+ if val.get("type") == "python" and "value" in val:
+ refs.append((prefix, int(val["value"])))
+ else:
+ for k_sub, v_sub in val.items():
+ sub_p = f"{prefix}.{k_sub}" if prefix else k_sub
+ refs.extend(_extract_refs(v_sub, sub_p))
+ elif isinstance(val, list):
+ for idx, item in enumerate(val):
+ sub_p = f"{prefix}[{idx}]"
+ refs.extend(_extract_refs(item, sub_p))
+ return refs
+
+ for oid, n in self.objects.items():
+ for it in n.init_tasks:
+ if it in self.objects:
+ n.outgoing_refs.append(("__init_task__", it))
+ self.objects[it].incoming_refs.append((oid, "__init_task__"))
+ for k, v in n.fields.items():
+ for rk, tid in _extract_refs(v, k):
+ if tid in self.objects:
+ n.outgoing_refs.append((rk, tid))
+ self.objects[tid].incoming_refs.append((oid, rk))
+
+ def resolve_path(self, spec: Any) -> Optional[ResolvedPath]:
+ """Attempt to resolve a parameter value as a path specification."""
+ if isinstance(spec, dict) and spec.get("type") in ("path", "path.serialized"):
+ return ResolvedPath.resolve(
+ spec.get("value", ""),
+ spec.get("base", "absolute"),
+ self.job_path,
+ self.workspace_path,
+ )
+ return None
+
+ def to_python_snippet(
+ self, oid: int, indent: int = 0, seen: Optional[Set[int]] = None
+ ) -> str:
+ """Synthesize a reproducible Python `.C(...)` configuration snippet."""
+ seen = seen or set()
+ node = self.objects.get(oid)
+ if not node:
+ return f"#{oid}"
+ if oid in seen:
+ return f"{node.type_name}.C(...) # cycle to #{oid}"
+ seen.add(oid)
+
+ sp, spi = " " * indent, " " * (indent + 1)
+ args = [
+ f"{spi}{k}={self._fmt_py(v, indent + 1, set(seen))},"
+ for k, v in sorted(node.fields.items())
+ ]
+ return f"{node.type_name}.C(\n" + "\n".join(args) + f"\n{sp})"
+
+ def _fmt_py(self, v: Any, indent: int, seen: Set[int]) -> str:
+ """Format an individual field value for Python code synthesis."""
+ if isinstance(v, dict):
+ t = v.get("type")
+ if t == "python":
+ return self.to_python_snippet(int(v["value"]), indent, seen)
+ if t in ("path", "path.serialized"):
+ b = v.get("base")
+ return f'Path("{v.get("value")}")' + (f" # base: {b}" if b else "")
+ if t == "enum":
+ return f"{v.get('enum')}.{v.get('value')}"
+ return (
+ "{"
+ + ", ".join(
+ f'"{k}": {self._fmt_py(val, indent, seen)}' for k, val in v.items()
+ )
+ + "}"
+ )
+ if isinstance(v, list):
+ return "[" + ", ".join(self._fmt_py(x, indent, seen) for x in v) + "]"
+ return repr(v)
diff --git a/src/experimaestro/tools/xpm_dag/__init__.py b/src/experimaestro/tools/xpm_dag/__init__.py
new file mode 100644
index 00000000..11a55635
--- /dev/null
+++ b/src/experimaestro/tools/xpm_dag/__init__.py
@@ -0,0 +1,63 @@
+"""Experimaestro DAG visualization, analysis, and export utilities."""
+
+from .draw_dag import (
+ main,
+ plot_dag,
+ print_robust_tree,
+ run_dag,
+)
+from .xpm_dag import (
+ BOX_H,
+ BOX_W,
+ MARGIN_X,
+ MARGIN_Y,
+ build_dag_json,
+ compute_dag_depths,
+ compute_dag_layout,
+ extract_config_tags,
+ filter_subgraph_by_config,
+ filter_tasks_only,
+ get_all_dependencies,
+ get_experiment_configs,
+ is_init_task_node,
+ load_dag_from_json,
+ load_run_graph,
+ load_run_status,
+ parse_filter_arg,
+ resolve_run_dir,
+ resolve_target_config,
+ stack_dag_nodes,
+)
+from .xpm_web_dag import (
+ generate_dag_html,
+ serve_dag_web,
+)
+
+__all__ = [
+ "BOX_H",
+ "BOX_W",
+ "MARGIN_X",
+ "MARGIN_Y",
+ "build_dag_json",
+ "compute_dag_depths",
+ "compute_dag_layout",
+ "extract_config_tags",
+ "filter_subgraph_by_config",
+ "filter_tasks_only",
+ "generate_dag_html",
+ "get_all_dependencies",
+ "get_experiment_configs",
+ "is_init_task_node",
+ "load_dag_from_json",
+ "load_run_graph",
+ "load_run_status",
+ "main",
+ "parse_filter_arg",
+ "plot_dag",
+ "print_robust_tree",
+ "resolve_run_dir",
+ "resolve_target_config",
+ "run_dag",
+ "serve_dag_web",
+ "stack_dag_nodes",
+]
diff --git a/src/experimaestro/tools/xpm_dag/__main__.py b/src/experimaestro/tools/xpm_dag/__main__.py
new file mode 100644
index 00000000..1e66431c
--- /dev/null
+++ b/src/experimaestro/tools/xpm_dag/__main__.py
@@ -0,0 +1,6 @@
+"""Entrypoint for `python -m experimaestro.tools.xpm_dag`."""
+
+from .draw_dag import main
+
+if __name__ == "__main__":
+ main()
diff --git a/src/experimaestro/tools/xpm_dag/dag_template.html b/src/experimaestro/tools/xpm_dag/dag_template.html
new file mode 100644
index 00000000..1cc1e8e2
--- /dev/null
+++ b/src/experimaestro/tools/xpm_dag/dag_template.html
@@ -0,0 +1,1894 @@
+
+
+
+
+
+ __TITLE__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Copied to clipboard
+
+
+
+
diff --git a/src/experimaestro/tools/xpm_dag/draw_dag.py b/src/experimaestro/tools/xpm_dag/draw_dag.py
new file mode 100644
index 00000000..2d1cb68b
--- /dev/null
+++ b/src/experimaestro/tools/xpm_dag/draw_dag.py
@@ -0,0 +1,865 @@
+"""Draw the dependency DAG of an Experimaestro experiment run as a tree, plot, or web page.
+
+Run with::
+
+ # Interactive DAG web visualization in browser:
+ uv run python -m experimaestro.tools.xpm_dag --web
+
+ # Export standalone interactive HTML report (auto-detected, zero-server):
+ uv run python -m experimaestro.tools.xpm_dag --save-as dag.html
+
+ # Export standardized JSON graph:
+ uv run python -m experimaestro.tools.xpm_dag --save-as dag.json
+
+ # Text tree terminal output:
+ uv run python -m experimaestro.tools.xpm_dag --no-plot
+
+ # Interactive matplotlib plot:
+ uv run python -m experimaestro.tools.xpm_dag
+
+ # Save plot directly to image without GUI window:
+ uv run python -m experimaestro.tools.xpm_dag --save-as dag.png
+
+ # List all hyperparameter configurations:
+ uv run python -m experimaestro.tools.xpm_dag --list-configs
+
+ # Inspect a single configuration:
+ uv run python -m experimaestro.tools.xpm_dag --single-config -c 0
+ uv run python -m experimaestro.tools.xpm_dag --filter "lr=0.001,kernel_size=3"
+
+This module provides the main CLI entrypoint and coordinates terminal ASCII,
+matplotlib, and interactive web representations of the experiment DAG.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from pathlib import Path
+import types
+import networkx as nx
+
+from experimaestro import tags
+
+# Re-export core DAG utilities and models for display styles & test compatibility
+from .xpm_dag import (
+ BOX_H,
+ BOX_W,
+ MARGIN_X,
+ MARGIN_Y,
+ build_dag_json,
+ compute_dag_depths,
+ compute_dag_layout,
+ extract_config_tags,
+ filter_subgraph_by_config,
+ filter_tasks_only,
+ get_all_dependencies,
+ get_experiment_configs,
+ is_init_task_node,
+ load_dag_from_json,
+ load_run_graph,
+ load_run_status,
+ parse_filter_arg,
+ resolve_run_dir,
+ resolve_target_config,
+ stack_dag_nodes,
+)
+from .xpm_web_dag import (
+ generate_dag_html,
+ serve_dag_web,
+)
+
+__all__ = [
+ "BOX_H",
+ "BOX_W",
+ "MARGIN_X",
+ "MARGIN_Y",
+ "build_dag_json",
+ "compute_dag_depths",
+ "compute_dag_layout",
+ "extract_config_tags",
+ "filter_subgraph_by_config",
+ "filter_tasks_only",
+ "generate_dag_html",
+ "get_all_dependencies",
+ "get_experiment_configs",
+ "is_init_task_node",
+ "load_dag_from_json",
+ "load_run_graph",
+ "load_run_status",
+ "parse_filter_arg",
+ "plot_dag",
+ "print_robust_tree",
+ "resolve_run_dir",
+ "resolve_target_config",
+ "run_dag",
+ "serve_dag_web",
+ "stack_dag_nodes",
+]
+
+
+def print_robust_tree(
+ job_ids: list[str],
+ dependents: dict[str, list[str]],
+ dependencies: dict[str, list[str]],
+ info_jobs: dict,
+ job_tags_dict: dict,
+ visited: set[str],
+ depths: dict[str, int],
+ pos: dict[str, tuple[float, float]],
+ prefix: str = "",
+ is_last: bool = True,
+ stack_members: dict[str, list[str]] | None = None,
+) -> None:
+ """Recursively print the dependency tree with individual nodes, depth and Y info."""
+ if not job_ids:
+ return
+
+ def job_sort_key(jid: str):
+ config = info_jobs.get(jid)
+ return (
+ depths.get(jid, 0),
+ -pos.get(jid, (0, 0))[1],
+ type(config).__name__ if config else "",
+ jid,
+ )
+
+ sorted_job_ids = sorted(job_ids, key=job_sort_key)
+
+ for idx, jid in enumerate(sorted_job_ids):
+ is_last_node = is_last and (idx == len(sorted_job_ids) - 1)
+ config = info_jobs.get(jid)
+ job_type = (
+ type(config).__name__.replace(".XPMConfig", "") if config else "Unknown"
+ )
+ depth = depths.get(jid, 0)
+ y_coord = pos.get(jid, (0, 0))[1]
+ is_init = is_init_task_node(config)
+ members = (stack_members or {}).get(jid, [jid])
+ stack_suffix = f" x {len(members)}" if len(members) > 1 else ""
+ init_badge = " [INIT]" if is_init else ""
+
+ # Use tags from jobs.jsonl, fallback to tags(config)
+ jtags = job_tags_dict.get(jid)
+ if not jtags and config:
+ try:
+ jtags = tags(config)
+ except Exception:
+ jtags = {}
+ jtags = jtags or {}
+ tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items())
+ label = (
+ f"{job_type}{init_badge}{stack_suffix} [{jid[:8]}] "
+ f"(depth {depth}, y {y_coord:.1f})"
+ )
+ if tag_str:
+ label += f" ({tag_str})"
+
+ already_visited = jid in visited
+ if already_visited:
+ label += " (already shown)"
+
+ connector = "└── " if is_last_node else "├── "
+ print(f"{prefix}{connector}{label}") # noqa: T201
+
+ if already_visited:
+ continue
+
+ visited.add(jid)
+ child_prefix = prefix + (" " if is_last_node else "│ ")
+ next_job_ids = sorted(dependents.get(jid, []), key=job_sort_key)
+ if next_job_ids:
+ print_robust_tree(
+ next_job_ids,
+ dependents,
+ dependencies,
+ info_jobs,
+ job_tags_dict,
+ visited,
+ depths,
+ pos,
+ child_prefix,
+ True,
+ stack_members=stack_members,
+ )
+
+
+def plot_dag(
+ full_jobs: dict,
+ job_tags_dict: dict,
+ pos: dict[str, tuple[float, float]],
+ reduced_dag: nx.DiGraph,
+ box_w: float = BOX_W,
+ box_h: float = BOX_H,
+ title_suffix: str = "",
+ show_plot: bool = True,
+ save_path: Path | None = None,
+ stack_members: dict[str, list[str]] | None = None,
+) -> None:
+ """Plot the DAG hierarchically using pre-computed layout and reduced graph.
+
+ Args:
+ full_jobs: Mapping of job ID to config/task instance.
+ job_tags_dict: Mapping of job ID to tags dictionary.
+ pos: Node coordinates mapping job ID to (x, y).
+ reduced_dag: Transitive reduction of the DAG.
+ box_w: Width of node boxes.
+ box_h: Height of node boxes.
+ title_suffix: Optional suffix appended to figure title (e.g. config details).
+ show_plot: Whether to display the plot interactively.
+ save_path: Optional path to save the figure image.
+ stack_members: Optional mapping of representative job ID to stacked member IDs.
+ """
+ try:
+ import matplotlib.pyplot as plt
+ from matplotlib.lines import Line2D
+ from matplotlib.patches import FancyBboxPatch
+ except ImportError:
+ logging.error(
+ "matplotlib is required for graphical plotting. "
+ "Please install it using: pip install matplotlib"
+ )
+ return
+
+ from experimaestro import Action, Task, tags
+
+ node_info = {}
+ for jid, config in full_jobs.items():
+ name = type(config).__name__.replace(".XPMConfig", "")
+ ntype = (
+ "action"
+ if isinstance(config, Action)
+ else "task"
+ if isinstance(config, Task)
+ else "init"
+ if is_init_task_node(config)
+ else "config"
+ )
+
+ # Get tags and create label
+ jtags = job_tags_dict.get(jid)
+ if not jtags and config:
+ try:
+ jtags = tags(config)
+ except Exception:
+ jtags = {}
+ jtags = jtags or {}
+ tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items())
+ members = (stack_members or {}).get(jid, [jid])
+
+ node_info[jid] = {
+ "name": f"{name} [INIT]" if ntype == "init" else name,
+ "hash": f"[{jid[:8]}]",
+ "tags": tag_str if tag_str else None,
+ "type": ntype,
+ "stack_count": len(members),
+ }
+
+ # Calculate dynamic figure size based on the layout
+ all_x = [p[0] for p in pos.values()]
+ all_y = [p[1] for p in pos.values()]
+ min_x, max_x = min(all_x), max(all_x)
+ min_y, max_y = min(all_y), max(all_y)
+
+ # Scale factor: 1 unit = 0.8 inches
+ scale = 0.8
+ fig_w = max(10, (max_x - min_x + box_w * 2) * scale)
+ fig_h = max(6, (max_y - min_y + box_h * 2) * scale)
+
+ plt.figure(figsize=(fig_w, fig_h))
+ ax = plt.gca()
+ title = "Experimaestro DAG (Hierarchical Top-Aligned Flow)"
+ if title_suffix:
+ title += f" - {title_suffix}"
+ plt.title(title)
+
+ for u, v in reduced_dag.edges():
+ (x1, y1), (x2, y2) = pos[u], pos[v]
+ plt.annotate(
+ "",
+ xy=(x2 - box_w / 2, y2),
+ xytext=(x1 + box_w / 2, y1),
+ zorder=1,
+ arrowprops=dict(
+ arrowstyle="->",
+ color="black",
+ alpha=1.0,
+ shrinkA=0,
+ shrinkB=0,
+ linewidth=1.5,
+ ),
+ )
+
+ styles = {
+ "config": "#e6ffed",
+ "task": "#e6f4ff",
+ "init": "#f3e8ff",
+ "action": "#fff1f0",
+ }
+ for jid, (x, y) in pos.items():
+ info = node_info[jid]
+ scount = info["stack_count"]
+ if scount > 1:
+ n_frames = 3 if scount > 5 else 2
+ for layer in range(n_frames, 0, -1):
+ dx = layer * 0.35
+ dy = -layer * 0.08
+ ax.add_patch(
+ FancyBboxPatch(
+ (x - box_w / 2 + dx, y - box_h / 2 + dy),
+ box_w,
+ box_h,
+ boxstyle="round,pad=0.03,rounding_size=0.2",
+ facecolor=styles[info["type"]],
+ edgecolor="#888888",
+ linewidth=1.4,
+ alpha=0.85,
+ zorder=1.5,
+ )
+ )
+ plt.text(
+ x + box_w / 2 + (n_frames * 0.35) + 0.3,
+ y + box_h / 2 - 0.1,
+ f"x {scount}",
+ ha="left",
+ va="center",
+ fontsize=9.5,
+ fontweight="bold",
+ color="#334155",
+ zorder=4,
+ )
+
+ ax.add_patch(
+ FancyBboxPatch(
+ (x - box_w / 2, y - box_h / 2),
+ box_w,
+ box_h,
+ boxstyle="round,pad=0.03,rounding_size=0.2",
+ facecolor=styles[info["type"]],
+ edgecolor="#7c3aed" if info["type"] == "init" else "#666666",
+ linewidth=2.0,
+ zorder=2,
+ )
+ )
+ if info["tags"]:
+ plt.text(
+ x,
+ y + 0.25,
+ info["name"],
+ ha="center",
+ va="center",
+ fontsize=10.5,
+ fontweight="bold",
+ color="#111111",
+ zorder=3,
+ )
+ plt.text(
+ x,
+ y + 0.03,
+ info["hash"],
+ ha="center",
+ va="center",
+ fontsize=7.5,
+ color="#777777",
+ zorder=3,
+ )
+ plt.text(
+ x,
+ y - 0.23,
+ info["tags"],
+ ha="center",
+ va="center",
+ fontsize=8.5,
+ fontweight="bold",
+ color="#222222",
+ zorder=3,
+ )
+ else:
+ plt.text(
+ x,
+ y + 0.14,
+ info["name"],
+ ha="center",
+ va="center",
+ fontsize=10.5,
+ fontweight="bold",
+ color="#111111",
+ zorder=3,
+ )
+ plt.text(
+ x,
+ y - 0.14,
+ info["hash"],
+ ha="center",
+ va="center",
+ fontsize=7.5,
+ color="#777777",
+ zorder=3,
+ )
+
+ plt.legend(
+ handles=[
+ Line2D(
+ [0],
+ [0],
+ marker="s",
+ color="w",
+ label=k.capitalize(),
+ markerfacecolor=v,
+ markersize=10,
+ markeredgecolor="gray",
+ )
+ for k, v in styles.items()
+ ],
+ loc="upper left",
+ )
+
+ plt.xlim(min_x - box_w, max_x + box_w)
+ plt.ylim(min_y - box_h, max_y + box_h)
+ plt.axis("off")
+ plt.tight_layout()
+ if save_path:
+ plt.savefig(save_path, bbox_inches="tight")
+ print(f"Saved DAG plot to {save_path}") # noqa: T201
+ if show_plot:
+ plt.show()
+ plt.close()
+
+
+def _resolve_save_as_destination(
+ save_as: Path,
+ html: Path | None,
+ json_out: str | None,
+ plot: bool,
+ web: bool | None,
+) -> tuple[Path | None, str | None, Path | None, bool, bool | None] | None:
+ """Resolve file formats and destination paths for --save-as."""
+ ext = save_as.suffix.lower()
+ save_plot_path = None
+ if ext in (".html", ".htm"):
+ html = save_as
+ web = False
+ elif ext == ".json":
+ json_out = str(save_as)
+ web = False
+ elif ext in (".png", ".pdf", ".svg", ".jpg", ".jpeg"):
+ save_plot_path = save_as
+ plot = False
+ web = False
+ else:
+ logging.error(
+ "Unsupported file extension '%s' for --save-as. "
+ "Supported extensions: .html, .htm, .json, .png, .pdf, .svg, .jpg",
+ ext,
+ )
+ return None
+ return html, json_out, save_plot_path, plot, web
+
+
+def _display_experiment_configs(run_dir: Path, exp_id: str, run_id: str) -> int:
+ """Display available configuration parameter combinations for an experiment run."""
+ configs = get_experiment_configs(run_dir)
+ if not configs:
+ print(f"No configuration tags found for experiment '{exp_id}' ({run_id}).") # noqa: T201
+ return 0
+ print(f"Available configurations in experiment '{exp_id}' (run {run_id}):") # noqa: T201
+ for idx, cfg_dict in enumerate(configs):
+ cfg_str = ", ".join(f"{k}={v}" for k, v in sorted(cfg_dict.items()))
+ print(f" [{idx}] {cfg_str}") # noqa: T201
+ return 0
+
+
+def _export_dag_json(json_out: str, dag_json: dict) -> None:
+ """Export DAG structure as formatted JSON."""
+ json_str = json.dumps(dag_json, indent=2)
+ if json_out:
+ out_path = Path(json_out)
+ out_path.parent.mkdir(parents=True, exist_ok=True)
+ out_path.write_text(json_str)
+ print(f"Saved DAG JSON to {out_path}") # noqa: T201
+ else:
+ print(json_str) # noqa: T201
+
+
+def _export_dag_html(html: Path, dag_json: dict, exp_id: str, layout: str) -> None:
+ """Export standalone interactive HTML visualization page."""
+ html_str = generate_dag_html(
+ dag_json,
+ title=f"Experimaestro DAG - {exp_id}",
+ default_layout=layout,
+ )
+ html.parent.mkdir(parents=True, exist_ok=True)
+ html.write_text(html_str)
+ print(f"Saved interactive DAG HTML to {html}") # noqa: T201
+
+
+def _display_dag_tree(
+ full_jobs: dict,
+ dependencies: dict[str, list[str]],
+ job_tags_dict: dict[str, dict],
+ stack: bool,
+) -> None:
+ """Render terminal text tree visualization."""
+ t_jobs, t_deps, t_tags = full_jobs, dependencies, job_tags_dict
+ stack_members = None
+ if stack:
+ t_jobs, t_deps, t_tags, stack_members = stack_dag_nodes(
+ full_jobs, dependencies, job_tags_dict
+ )
+ dependents: dict[str, list[str]] = {}
+ for jid, deps in t_deps.items():
+ for d in deps:
+ dependents.setdefault(d, []).append(jid)
+
+ depths, pos, _ = compute_dag_layout(t_jobs, t_deps, dependents)
+ tree_roots = [jid for jid in t_jobs if not t_deps.get(jid)]
+ print_robust_tree(
+ tree_roots,
+ dependents,
+ t_deps,
+ t_jobs,
+ t_tags,
+ set(),
+ depths,
+ pos,
+ stack_members=stack_members,
+ )
+
+
+def _display_or_save_dag_plot(
+ full_jobs: dict,
+ dependencies: dict[str, list[str]],
+ job_tags_dict: dict[str, dict],
+ title_suffix: str,
+ plot: bool,
+ save_plot_path: Path | None,
+ stack: bool,
+) -> None:
+ """Render graphical matplotlib DAG plot or save it to image/vector file."""
+ p_jobs, p_deps, p_tags = full_jobs, dependencies, job_tags_dict
+ stack_members = None
+ if stack:
+ p_jobs, p_deps, p_tags, stack_members = stack_dag_nodes(
+ full_jobs, dependencies, job_tags_dict
+ )
+ dependents: dict[str, list[str]] = {}
+ for jid, deps in p_deps.items():
+ for d in deps:
+ dependents.setdefault(d, []).append(jid)
+
+ depths, pos, reduced_dag = compute_dag_layout(p_jobs, p_deps, dependents)
+ plot_dag(
+ p_jobs,
+ p_tags,
+ pos,
+ reduced_dag,
+ title_suffix=title_suffix,
+ show_plot=plot,
+ save_path=save_plot_path,
+ stack_members=stack_members,
+ )
+
+
+def run_dag(
+ *,
+ experiment_id: str | None = None,
+ run_id: str | None = None,
+ workspace: str | None = None,
+ web: bool | None = None,
+ port: int = 0,
+ no_open: bool = False,
+ tree: bool = False,
+ plot: bool = False,
+ save_as: Path | None = None,
+ html: Path | None = None,
+ json_out: str | None = None,
+ tasks_only: bool = False,
+ list_configs: bool = False,
+ single_config: bool = False,
+ config_index: int | None = None,
+ filter_tags: str | None = None,
+ include_actions: bool = False,
+ layout: str = "dagre",
+ stack: bool = True,
+) -> int:
+ """Execute the experiment DAG visualization/export pipeline with given options."""
+ save_plot_path = None
+ if save_as:
+ resolved = _resolve_save_as_destination(save_as, html, json_out, plot, web)
+ if resolved is None:
+ return 1
+ html, json_out, save_plot_path, plot, web = resolved
+
+ # Resolve default web behavior: enabled by default unless --tree, --plot, or file export is active
+ if web is None:
+ web = not (
+ plot or tree or save_as or html or json_out is not None or list_configs
+ )
+
+ run_info = resolve_run_dir(workspace, experiment_id, run_id)
+ if run_info is None:
+ return 1
+ run_id, run_dir = run_info
+ exp_id = run_dir.parent.name
+
+ if list_configs:
+ return _display_experiment_configs(run_dir, exp_id, run_id)
+
+ graph_data = load_run_graph(run_dir)
+ if graph_data is None:
+ return 1
+ full_jobs, dependencies, job_tags_dict = graph_data
+
+ title_suffix = ""
+ if single_config or config_index is not None or filter_tags:
+ configs = get_experiment_configs(run_dir)
+ filter_args = types.SimpleNamespace(
+ config_index=config_index,
+ filter=filter_tags,
+ single_config=single_config,
+ )
+ target_tags, title_suffix = resolve_target_config(filter_args, configs)
+ if target_tags is None:
+ return 1
+ full_jobs, dependencies = filter_subgraph_by_config(
+ full_jobs,
+ dependencies,
+ job_tags_dict,
+ target_tags,
+ include_actions=include_actions,
+ )
+
+ if tasks_only:
+ full_jobs, dependencies = filter_tasks_only(full_jobs, dependencies)
+
+ workspace_path = str(run_dir.parents[2]) if len(run_dir.parents) >= 3 else ""
+
+ # Export JSON if requested
+ if json_out is not None:
+ dag_json = build_dag_json(
+ full_jobs,
+ dependencies,
+ job_tags_dict,
+ run_dir=run_dir,
+ experiment_id=exp_id,
+ run_id=run_id,
+ workspace=workspace_path,
+ )
+ _export_dag_json(json_out, dag_json)
+ return 0
+
+ # Export standalone HTML page if requested
+ if html is not None:
+ dag_json = build_dag_json(
+ full_jobs,
+ dependencies,
+ job_tags_dict,
+ run_dir=run_dir,
+ experiment_id=exp_id,
+ run_id=run_id,
+ workspace=workspace_path,
+ )
+ _export_dag_html(html, dag_json, exp_id, layout)
+ return 0
+
+ # Optional terminal text tree visualization
+ if tree:
+ _display_dag_tree(full_jobs, dependencies, job_tags_dict, stack)
+ if not (web or plot or save_plot_path):
+ return 0
+
+ # Launch interactive web server (default mode)
+ if web:
+ dag_json = build_dag_json(
+ full_jobs,
+ dependencies,
+ job_tags_dict,
+ run_dir=run_dir,
+ experiment_id=exp_id,
+ run_id=run_id,
+ workspace=workspace_path,
+ )
+ serve_dag_web(
+ dag_json,
+ port=port,
+ open_browser=not no_open,
+ default_layout=layout,
+ )
+ return 0
+
+ # Optional Matplotlib & graphical plot visualization (requires matplotlib)
+ if plot or save_plot_path:
+ _display_or_save_dag_plot(
+ full_jobs,
+ dependencies,
+ job_tags_dict,
+ title_suffix,
+ plot,
+ save_plot_path,
+ stack,
+ )
+ return 0
+
+ return 0
+
+
+def main() -> None:
+ logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "experiment",
+ nargs="?",
+ default=None,
+ help="Experiment identifier (optional positional).",
+ )
+ parser.add_argument(
+ "--workspace",
+ "-w",
+ type=str,
+ default=None,
+ help="Workspace ID or directory path (default: auto-detected via find_workspace).",
+ )
+ parser.add_argument(
+ "--experiment-id",
+ "-e",
+ default=None,
+ help="Experiment identifier.",
+ )
+ parser.add_argument(
+ "--run-id",
+ "-r",
+ default=None,
+ help="Specific run timestamp ID (default: most recent run).",
+ )
+ parser.add_argument(
+ "--tasks-only",
+ action="store_true",
+ help="Only display Task and Action nodes, skipping static config nodes.",
+ )
+ parser.add_argument(
+ "--web",
+ "-W",
+ action="store_true",
+ default=None,
+ help="Launch interactive DAG web viewer in browser (default).",
+ )
+ parser.add_argument(
+ "--no-web",
+ action="store_false",
+ dest="web",
+ help="Do not launch interactive DAG web viewer.",
+ )
+ parser.add_argument(
+ "--tree",
+ "-t",
+ action="store_true",
+ help="Print text tree representation of the DAG to terminal.",
+ )
+ parser.add_argument(
+ "--plot",
+ action="store_true",
+ default=False,
+ help="Display matplotlib interactive plot window (requires matplotlib).",
+ )
+ parser.add_argument(
+ "--port",
+ type=int,
+ default=0,
+ help="Port for web server (default: random available port).",
+ )
+ parser.add_argument(
+ "--no-open",
+ action="store_true",
+ help="Do not automatically open browser when launching web viewer.",
+ )
+ parser.add_argument(
+ "--html",
+ type=Path,
+ default=None,
+ help="Path where the standalone interactive DAG HTML page should be saved (e.g. dag.html).",
+ )
+ parser.add_argument(
+ "--json",
+ nargs="?",
+ const="",
+ default=None,
+ help="Export DAG as JSON to stdout or file (compatible with Experimaestro web CLI).",
+ )
+ parser.add_argument(
+ "--list-configs",
+ action="store_true",
+ help="List all unique configuration parameter sets found in the experiment and exit.",
+ )
+ parser.add_argument(
+ "--single-config",
+ action="store_true",
+ help="Render the DAG for a single grid configuration (defaults to config index 0).",
+ )
+ parser.add_argument(
+ "-c",
+ "--config-index",
+ type=int,
+ default=None,
+ help="Index of the configuration to display (see --list-configs). Implies --single-config.",
+ )
+ parser.add_argument(
+ "--filter",
+ default=None,
+ help="Filter the DAG by tag values (e.g. 'lr=0.001,kernel_size=3').",
+ )
+ parser.add_argument(
+ "--include-actions",
+ action="store_true",
+ help="Include downstream Action and EvaluatedModel nodes when filtering by configuration.",
+ )
+ parser.add_argument(
+ "--layout",
+ choices=["dagre", "aligned"],
+ default="dagre",
+ help="Layout algorithm for interactive web view: 'dagre' (default) or 'aligned' (stage-aligned flow).",
+ )
+ parser.add_argument(
+ "--no-stack",
+ action="store_false",
+ dest="stack",
+ default=True,
+ help="Disable stacking of structurally equivalent nodes.",
+ )
+ parser.add_argument(
+ "--save-as",
+ type=Path,
+ default=None,
+ help="Path where the DAG output should be saved (auto-detects .html, .json, .png, etc.). "
+ "If ending in .html, generates standalone HTML without launching the server.",
+ )
+ args = parser.parse_args()
+
+ exp_id = args.experiment or args.experiment_id
+
+ run_dag(
+ experiment_id=exp_id,
+ run_id=args.run_id,
+ workspace=args.workspace,
+ web=args.web,
+ port=args.port,
+ no_open=args.no_open,
+ tree=args.tree,
+ plot=args.plot,
+ save_as=args.save_as,
+ html=args.html,
+ json_out=args.json,
+ tasks_only=args.tasks_only,
+ list_configs=args.list_configs,
+ single_config=args.single_config,
+ config_index=args.config_index,
+ filter_tags=args.filter,
+ include_actions=args.include_actions,
+ layout=args.layout,
+ stack=args.stack,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/experimaestro/tools/xpm_dag/xpm_dag.py b/src/experimaestro/tools/xpm_dag/xpm_dag.py
new file mode 100644
index 00000000..0f3360f0
--- /dev/null
+++ b/src/experimaestro/tools/xpm_dag/xpm_dag.py
@@ -0,0 +1,1215 @@
+"""Core DAG utilities, graph algorithms, and standardized JSON serialization.
+
+This module provides display-agnostic utilities for reading Experimaestro
+experiment graphs from workspace records (``objects.jsonl``, ``jobs.jsonl``,
+and ``status.json``), computing hierarchical forward topological layouts,
+filtering subgraphs by hyperparameter configurations, and serializing the DAG
+into a standardized dual-compatible JSON format.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from pathlib import Path
+import re
+import sys
+from typing import Any
+import networkx as nx
+
+from experimaestro import Action, Config, LightweightTask, Task, tags
+from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider
+from experimaestro.settings import find_workspace, get_workspace
+from experimaestro.tools.inspect_params.models import (
+ ParamsGraph,
+ resolve_serialized_value,
+)
+
+# Default layout sizing constants
+BOX_W = 25.0
+BOX_H = 1.05
+MARGIN_X = 4.0 # Gap between boxes horizontally
+MARGIN_Y = 0.55 # Gap between boxes vertically
+
+
+def resolve_run_dir(
+ workspace_arg: str | Path | None,
+ experiment_id: str | None = None,
+ run_id: str | None = None,
+) -> tuple[str, Path] | None:
+ """Find and return (run_id, run_dir) from the workspace.
+
+ Uses the explicitly specified workspace if provided (by ID or path).
+ Otherwise, auto-selects the default workspace just as in real experiments
+ using Experimaestro's workspace triggers matching the experiment ID.
+ """
+ if workspace_arg:
+ ws_id = str(workspace_arg)
+ if get_workspace(ws_id) is not None:
+ ws_env = find_workspace(workspace=ws_id, experiment_id=experiment_id)
+ else:
+ ws_path = Path(workspace_arg).expanduser()
+ ws_env = find_workspace(workdir=ws_path, experiment_id=experiment_id)
+ else:
+ ws_env = find_workspace(experiment_id=experiment_id)
+
+ if getattr(ws_env, "is_remote", False):
+ from experimaestro.utils.remote import find_remote_workspace_experiment_run
+
+ try:
+ return find_remote_workspace_experiment_run(ws_env, experiment_id, run_id)
+ except Exception as e:
+ logging.error("Failed to resolve remote experiment run: %s", e)
+ return None
+
+ workspace = ws_env.path
+ experiments_dir = workspace / "experiments"
+ if not experiments_dir.exists():
+ logging.error("No 'experiments' directory found in workspace '%s'.", workspace)
+ return None
+
+ available_xps = sorted(
+ [
+ p.name
+ for p in experiments_dir.iterdir()
+ if p.is_dir() and not p.name.startswith(".")
+ ]
+ )
+ if not experiment_id:
+ if len(available_xps) == 1:
+ experiment_id = available_xps[0]
+ logging.info(
+ "Auto-selected single experiment '%s' in workspace '%s'.",
+ experiment_id,
+ workspace,
+ )
+ elif len(available_xps) > 1:
+ xps_str = ", ".join(f"'{x}'" for x in available_xps)
+ logging.error(
+ "No experiment specified. Please specify an experiment ID. Available experiments in workspace '%s': %s",
+ workspace,
+ xps_str,
+ )
+ return None
+ else:
+ logging.error("No experiments found in workspace '%s'.", workspace)
+ return None
+
+ exp_dir = experiments_dir / experiment_id
+ if not exp_dir.exists():
+ xps_str = (
+ ", ".join(f"'{x}'" for x in available_xps) if available_xps else "(none)"
+ )
+ logging.error(
+ "Experiment '%s' not found in workspace '%s'. Available experiments: %s",
+ experiment_id,
+ workspace,
+ xps_str,
+ )
+ return None
+
+ available_runs = sorted(
+ [p.name for p in exp_dir.iterdir() if p.is_dir() and not p.name.startswith(".")]
+ )
+ if not available_runs:
+ logging.error(
+ "Experiment '%s' in workspace '%s' has no recorded runs.",
+ experiment_id,
+ workspace,
+ )
+ return None
+
+ provider = WorkspaceStateProvider(workspace)
+ if run_id:
+ if run_id not in available_runs:
+ runs_str = ", ".join(f"'{r}'" for r in available_runs)
+ logging.error(
+ "Run '%s' not found for experiment '%s' in workspace '%s'. Available runs: %s",
+ run_id,
+ experiment_id,
+ workspace,
+ runs_str,
+ )
+ return None
+ resolved_run_id = run_id
+ else:
+ try:
+ current_run = provider.get_current_run(experiment_id)
+ except Exception:
+ current_run = None
+ resolved_run_id = current_run or available_runs[-1]
+
+ run_dir = exp_dir / resolved_run_id
+ if not run_dir.exists():
+ logging.error("Run directory does not exist: %s", run_dir)
+ return None
+
+ return resolved_run_id, run_dir
+
+
+def extract_config_tags(
+ config: Any, grid_keys: set[str] | None = None
+) -> dict[str, Any]:
+ """Extract hyperparameter tags from a Config node's xpmvalues.
+
+ If grid_keys is provided, extracts parameters matching grid_keys.
+ Otherwise, extracts all scalar primitive parameters (int, float, str, bool).
+ """
+ if config is None:
+ return {}
+ xpm = getattr(config, "__xpm__", None)
+ if xpm is None or not hasattr(xpm, "xpmvalues"):
+ return {}
+
+ raw_params: dict[str, Any] = {}
+ try:
+ for param, val in xpm.xpmvalues():
+ p_name = getattr(param, "name", str(param))
+ if isinstance(val, (int, float, str, bool)) and not p_name.startswith("_"):
+ raw_params[p_name] = val
+ except Exception:
+ return {}
+
+ if grid_keys:
+ return {k: v for k, v in raw_params.items() if k in grid_keys}
+ return raw_params
+
+
+class _LightweightParam:
+ """Minimal parameter descriptor carrying the argument name."""
+
+ __slots__ = ("name",)
+
+ def __init__(self, name: str):
+ self.name = name
+
+
+class _LightweightXPMInfo:
+ """Lightweight metadata container matching ConfigInformation's DAG interface."""
+
+ __slots__ = (
+ "identifier",
+ "task",
+ "init_tasks",
+ "dependencies",
+ "values",
+ "is_init_task",
+ )
+
+ def __init__(self, identifier: str, is_init_task: bool = False):
+ self.identifier = identifier
+ self.task: Any = None
+ self.init_tasks: list[Any] = []
+ self.dependencies: list[Any] = []
+ self.values: dict[str, Any] = {}
+ self.is_init_task = is_init_task
+
+ def xpmvalues(self):
+ for k, v in self.values.items():
+ yield _LightweightParam(k), v
+
+ def tags(self) -> dict[str, Any]:
+ return {}
+
+
+def is_init_task_node(config: Any) -> bool:
+ """Return True if a node is a LightweightTask (init_task) and not a full Task or Action."""
+ if config is None:
+ return False
+ if isinstance(config, (Task, Action)):
+ return False
+ if isinstance(config, LightweightTask):
+ return True
+ xpm = getattr(config, "__xpm__", None)
+ return bool(getattr(xpm, "is_init_task", False))
+
+
+def _load_lightweight_objects(
+ graph: ParamsGraph,
+ job_ids: set[str],
+ action_ids: set[str],
+) -> dict[int, Any]:
+ """Construct lightweight Config, LightweightTask, Task, and Action nodes from an indexed ParamsGraph.
+
+ Avoids importing external Python modules or executing remote file paths while
+ preserving class names, isinstance(node, (Task, Action, LightweightTask, Config)) semantics,
+ and __xpm__ dependency/parameter traversal.
+ """
+ cls_cache: dict[tuple[type, str, str], type] = {}
+ objects: dict[int, Any] = {}
+
+ for obj_id, obj_node in graph.objects.items():
+ ident = obj_node.identifier
+ module_name = obj_node.module or "_xpm_dag_"
+ type_name = obj_node.type_name or "Unknown"
+
+ if ident and ident in action_ids:
+ base_cls: type = Action
+ elif (ident and ident in job_ids) or obj_node.is_task:
+ base_cls = Task
+ elif obj_node.is_init_task:
+ base_cls = LightweightTask
+ else:
+ base_cls = Config
+ # Fallback to already-imported module in sys.modules if present
+ loaded_mod = sys.modules.get(module_name)
+ if loaded_mod is not None:
+ real_cls = getattr(loaded_mod, type_name, None)
+ if isinstance(real_cls, type):
+ if issubclass(real_cls, Action):
+ base_cls = Action
+ elif issubclass(real_cls, Task):
+ base_cls = Task
+ elif issubclass(real_cls, LightweightTask):
+ base_cls = LightweightTask
+
+ cls_key = (base_cls, module_name, type_name)
+ cls = cls_cache.get(cls_key)
+ if cls is None:
+ cls = type(type_name, (base_cls,), {"__module__": module_name})
+ cls_cache[cls_key] = cls
+
+ node = object.__new__(cls)
+ object.__setattr__(
+ node,
+ "__xpm__",
+ _LightweightXPMInfo(ident, is_init_task=(base_cls is LightweightTask)),
+ )
+ objects[obj_id] = node
+
+ for obj_id, obj_node in graph.objects.items():
+ node = objects[obj_id]
+ xpm: _LightweightXPMInfo = node.__xpm__
+
+ if (
+ obj_node.task_id is not None
+ and obj_node.task_id in objects
+ and obj_node.task_id != obj_id
+ ):
+ xpm.task = objects[obj_node.task_id]
+
+ if obj_node.init_tasks:
+ xpm.init_tasks = [
+ objects[tid] for tid in obj_node.init_tasks if tid in objects
+ ]
+
+ for fname, fval in obj_node.fields.items():
+ xpm.values[fname] = resolve_serialized_value(fval, objects)
+
+ return objects
+
+
+def load_run_graph(
+ run_dir: Path,
+) -> tuple[dict, dict[str, list[str]], dict[str, dict]] | None:
+ """Load objects.jsonl and jobs.jsonl into full_jobs, dependencies, and job_tags_dict."""
+ objects_file = run_dir / "objects.jsonl"
+ if not objects_file.exists():
+ logging.error("Objects record not found: %s", objects_file)
+ return None
+
+ graph = ParamsGraph.from_file(objects_file, load_docstrings=False)
+
+ job_tags_dict: dict[str, dict] = {}
+ job_ids: set[str] = set()
+ grid_keys: set[str] = set()
+ jobs_jsonl = run_dir / "jobs.jsonl"
+ if jobs_jsonl.exists():
+ with jobs_jsonl.open() as f:
+ for line in f:
+ if line.strip():
+ job_data = json.loads(line)
+ jid = job_data.get("job_id")
+ if jid:
+ job_ids.add(jid)
+ if "tags" in job_data:
+ jtags = job_data["tags"] or {}
+ job_tags_dict[jid] = jtags
+ if jtags:
+ grid_keys.update(jtags.keys())
+
+ status_data = load_run_status(run_dir)
+ action_ids: set[str] = set(status_data.get("actions", {}).keys())
+
+ all_configs_dict = _load_lightweight_objects(
+ graph, job_ids=job_ids, action_ids=action_ids
+ )
+
+ full_jobs = {
+ obj_node.identifier: all_configs_dict[obj_id]
+ for obj_id, obj_node in graph.objects.items()
+ if obj_node.identifier
+ }
+
+ direct_deps = {
+ jid: get_all_dependencies(config, full_jobs, recursive=False)
+ for jid, config in full_jobs.items()
+ }
+ raw_all_deps = {
+ jid: set(get_all_dependencies(config, full_jobs, recursive=True))
+ for jid, config in full_jobs.items()
+ }
+ # Remove PathGenerator back-edges on direct dependencies where a Config's
+ # generated path points to the job directory of an enclosing Task/Config.
+ for jid, deps in list(direct_deps.items()):
+ if not isinstance(full_jobs.get(jid), (Task, Action)):
+ direct_deps[jid] = [
+ d for d in deps if jid not in raw_all_deps.get(d, set())
+ ]
+
+ # Expand transitive Config dependencies from cleaned direct_deps so nested
+ # Configs (e.g. ValidationSettings -> AggregatorValidationListener) never
+ # inherit stripped PathGenerator back-edges.
+ dependencies: dict[str, list[str]] = {}
+ for jid in full_jobs:
+ expanded: list[str] = []
+ seen_deps: set[str] = set()
+ queue = list(direct_deps.get(jid, []))
+ visited_cfg: set[str] = set()
+ while queue:
+ d = queue.pop(0)
+ if d == jid:
+ continue
+ if d not in seen_deps:
+ seen_deps.add(d)
+ expanded.append(d)
+ d_obj = full_jobs.get(d)
+ if (
+ d_obj is not None
+ and not isinstance(d_obj, (Task, Action, LightweightTask))
+ and not is_init_task_node(d_obj)
+ and d not in visited_cfg
+ ):
+ visited_cfg.add(d)
+ for sub_d in direct_deps.get(d, []):
+ if sub_d not in seen_deps:
+ queue.append(sub_d)
+ if expanded:
+ dependencies[jid] = expanded
+
+ # Populate tags for Config nodes (e.g. CNN architecture params)
+ for jid, config in full_jobs.items():
+ if jid not in job_tags_dict and config:
+ cfg_tags = extract_config_tags(config, grid_keys if grid_keys else None)
+ if cfg_tags:
+ job_tags_dict[jid] = cfg_tags
+
+ return full_jobs, dependencies, job_tags_dict
+
+
+def load_run_status(run_dir: Path) -> dict[str, Any]:
+ """Load status.json if present in the run directory."""
+ status_file = run_dir / "status.json"
+ if not status_file.exists():
+ return {}
+ try:
+ with status_file.open() as f:
+ return json.load(f)
+ except Exception as e:
+ logging.warning("Could not read status.json at %s: %s", status_file, e)
+ return {}
+
+
+def get_all_dependencies(
+ config, full_jobs: dict, *, recursive: bool = True
+) -> list[str]:
+ """Walk the config and its fields to find all dependencies."""
+ xpm = config.__xpm__
+ dep_ids = set()
+ current_id = str(xpm.identifier)
+
+ if hasattr(xpm, "task") and xpm.task:
+ dep_ids.add(str(xpm.task.__xpm__.identifier))
+ if hasattr(xpm, "init_tasks"):
+ for init_task in xpm.init_tasks:
+ if hasattr(init_task, "__xpm__"):
+ dep_ids.add(str(init_task.__xpm__.identifier))
+
+ for dep in getattr(xpm, "dependencies", []):
+ if hasattr(dep, "__identifier__"):
+ dep_ids.add(str(dep.__identifier__()))
+ elif hasattr(dep, "__xpm__"):
+ dep_ids.add(str(dep.__xpm__.identifier))
+ else:
+ dep_ids.add(str(dep))
+
+ processed_configs = set()
+
+ def walk_value(v):
+ from experimaestro import Config
+
+ if isinstance(v, Config):
+ v_id = str(v.__xpm__.identifier)
+ if v_id in full_jobs:
+ dep_ids.add(v_id)
+ if not recursive:
+ return
+ if v_id not in processed_configs:
+ processed_configs.add(v_id)
+ for _, sub_value in v.__xpm__.xpmvalues():
+ walk_value(sub_value)
+ elif isinstance(v, list):
+ for item in v:
+ walk_value(item)
+ elif isinstance(v, dict):
+ for item in v.values():
+ walk_value(item)
+ elif isinstance(v, (str, Path)) and "/jobs/" in str(v):
+ match = re.search(r"/jobs/[^/]+/([0-9a-f]{64})", str(v))
+ if match:
+ v_id = match.group(1)
+ if v_id in full_jobs:
+ dep_ids.add(v_id)
+
+ for name, value in xpm.xpmvalues():
+ walk_value(value)
+ if current_id in dep_ids:
+ dep_ids.remove(current_id)
+ return list(dep_ids)
+
+
+def _detect_bottleneck_configs(
+ job_records: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ """Detect core bottleneck grid-search configurations from jobs.jsonl records.
+
+ When an experiment contains both core model grid-search tasks (e.g. Learner
+ tagged with ``{base, lr, seed}``) and dataset/evaluation fan-out tasks
+ (e.g. IndexBuilder tagged with ``{index_documents}``, 1st-stage Evaluate
+ tagged with ``{dataset, first_stage}``, and 2nd-stage Evaluate tagged with
+ ``{base, lr, seed, dataset, first_stage, checkpoint}``), this function
+ identifies the bottleneck grid-search schema so that configuration filtering
+ selects one complete model run (including all its evaluation datasets) rather
+ than individual dataset jobs.
+ """
+ tagged_records = [r for r in job_records if r.get("tags")]
+ if not tagged_records:
+ return []
+
+ by_schema: dict[frozenset[str], list[dict[str, Any]]] = {}
+ schema_task_ids: dict[frozenset[str], set[str]] = {}
+ for r in tagged_records:
+ tdict = r["tags"]
+ schema = frozenset(tdict.keys())
+ by_schema.setdefault(schema, []).append(tdict)
+ if r.get("task_id"):
+ schema_task_ids.setdefault(schema, set()).add(r["task_id"])
+
+ schemas = list(by_schema.keys())
+ if len(schemas) > 1:
+ # Look for shared bottleneck schemas across distinct task stages via
+ # pairwise schema intersections I = S_a ∩ S_b (which covers both strict
+ # subsets S_a ⊂ S_b like Learner -> Evaluate and non-subset stage overlaps
+ # like IndexBuilder{model, doc} -> Evaluate{model, dataset}).
+ candidate_configs: dict[frozenset[str], list[dict[str, Any]]] = {}
+ for idx_a, s_a in enumerate(schemas):
+ tids_a = schema_task_ids.get(s_a, set())
+ uniq_a = {tuple(sorted(d.items())) for d in by_schema[s_a]}
+ for s_b in schemas[idx_a + 1 :]:
+ tids_b = schema_task_ids.get(s_b, set())
+ if tids_a and tids_b and tids_a == tids_b:
+ continue
+ inter = s_a & s_b
+ if not inter:
+ continue
+ uniq_b = {tuple(sorted(d.items())) for d in by_schema[s_b]}
+ proj_a = {
+ tuple(sorted((k, d[k]) for k in inter)) for d in by_schema[s_a]
+ }
+ proj_b = {
+ tuple(sorted((k, d[k]) for k in inter)) for d in by_schema[s_b]
+ }
+ shared_proj = proj_a & proj_b
+ union_proj = proj_a | proj_b
+ # Require shared bottleneck configurations across stages and a 1->N fan-out
+ if shared_proj and max(len(uniq_a), len(uniq_b)) > len(union_proj):
+ if inter not in candidate_configs:
+ configs_for_inter: list[dict[str, Any]] = []
+ seen_inter: set[tuple] = set()
+ for d in by_schema[s_a] + by_schema[s_b]:
+ proj_dict = {k: d[k] for k in inter}
+ key = tuple(sorted(proj_dict.items()))
+ if key in shared_proj and key not in seen_inter:
+ seen_inter.add(key)
+ configs_for_inter.append(proj_dict)
+ if configs_for_inter:
+ candidate_configs[inter] = configs_for_inter
+
+ if candidate_configs:
+ # Prefer largest bottleneck schema key-set, breaking ties by strongest compression (fewest configs)
+ target_schema = max(
+ candidate_configs.keys(),
+ key=lambda s: (len(s), -len(candidate_configs[s])),
+ )
+ return candidate_configs[target_schema]
+
+ # Default: deduplicate all non-empty tag dictionaries in order
+ configs = []
+ seen = set()
+ for r in tagged_records:
+ tdict = r["tags"]
+ key = tuple(sorted(tdict.items()))
+ if key not in seen:
+ seen.add(key)
+ configs.append(tdict)
+ return configs
+
+
+def get_experiment_configs(run_dir: Path) -> list[dict]:
+ """Retrieve unique bottleneck grid-search tag dictionaries from jobs.jsonl in a run directory."""
+ jobs_jsonl = run_dir / "jobs.jsonl"
+ if not jobs_jsonl.exists():
+ return []
+
+ job_records: list[dict[str, Any]] = []
+ with jobs_jsonl.open() as f:
+ for line in f:
+ if not line.strip():
+ continue
+ job_records.append(json.loads(line))
+
+ configs = _detect_bottleneck_configs(job_records)
+ if configs:
+ param_keys = sorted(set().union(*(c.keys() for c in configs)))
+ logging.info(
+ "Detected %d grid search configuration(s) (parameters: %s)",
+ len(configs),
+ ", ".join(param_keys),
+ )
+ return configs
+
+
+def parse_filter_arg(filter_str: str) -> dict[str, str | int | float]:
+ """Parse a comma-separated key=val filter string into typed values."""
+ parsed: dict[str, str | int | float] = {}
+ for item in filter_str.split(","):
+ item = item.strip()
+ if not item or "=" not in item:
+ continue
+ k, v = item.split("=", 1)
+ k, v = k.strip(), v.strip()
+ try:
+ parsed[k] = int(v)
+ except ValueError:
+ try:
+ parsed[k] = float(v)
+ except ValueError:
+ parsed[k] = v
+ return parsed
+
+
+def filter_subgraph_by_config(
+ full_jobs: dict,
+ dependencies: dict[str, list[str]],
+ job_tags_dict: dict[str, dict],
+ target_tags: dict,
+ include_actions: bool = False,
+) -> tuple[dict, dict[str, list[str]]]:
+ """Extract a subgraph for a single configuration or a subset of configurations."""
+ # 1. Identify jobs whose tags match all key-values in target_tags
+ matching = [
+ jid
+ for jid, tags_val in job_tags_dict.items()
+ if jid in full_jobs
+ and all(tags_val.get(k) == v for k, v in target_tags.items())
+ ]
+ if not matching:
+ return {}, {}
+
+ # 2. If filtering matched only upstream tasks (no matched task depends on another
+ # matched task, e.g. --filter index_documents=...), traverse downstream Task
+ # dependents whose tags do not conflict with target_tags.
+ from experimaestro import Action
+
+ selected_nodes = set(matching)
+ matched_tasks = [jid for jid in matching if isinstance(full_jobs.get(jid), Task)]
+ matched_task_set = set(matched_tasks)
+ has_multi_stage_match = any(
+ any(d in matched_task_set for d in dependencies.get(jid, []))
+ for jid in matched_tasks
+ )
+ if matched_tasks and not has_multi_stage_match:
+ dependents: dict[str, list[str]] = {}
+ for jid, deps in dependencies.items():
+ if jid not in full_jobs:
+ continue
+ for d in deps:
+ dependents.setdefault(d, []).append(jid)
+
+ down_stack = list(matched_tasks)
+ while down_stack:
+ curr = down_stack.pop()
+ for child in dependents.get(curr, []):
+ if child in selected_nodes:
+ continue
+ cfg = full_jobs.get(child)
+ if not isinstance(cfg, Task):
+ continue
+ child_tags = job_tags_dict.get(child) or {}
+ if all(
+ child_tags.get(k) is None or child_tags.get(k) == v
+ for k, v in target_tags.items()
+ ):
+ selected_nodes.add(child)
+ down_stack.append(child)
+
+ # 3. Traverse upstream ancestors (CNN, Datasets, ParameterLoader, FilterUnassessedTopics)
+ stack = list(selected_nodes)
+ while stack:
+ curr = stack.pop()
+ for parent in dependencies.get(curr, []):
+ if parent in full_jobs and parent not in selected_nodes:
+ selected_nodes.add(parent)
+ stack.append(parent)
+
+ # 4. If requested, include downstream actions and evaluated models that depend on matching jobs
+ if include_actions:
+ for jid, config in full_jobs.items():
+ if isinstance(config, Action) or type(config).__name__.startswith(
+ "EvaluatedModel"
+ ):
+ deps = dependencies.get(jid, [])
+ if any(m in deps for m in selected_nodes):
+ selected_nodes.add(jid)
+
+ # 4. Construct filtered jobs and dependencies restricted to selected_nodes
+ filtered_jobs = {jid: full_jobs[jid] for jid in selected_nodes}
+ filtered_dependencies = {
+ jid: [p for p in dependencies.get(jid, []) if p in selected_nodes]
+ for jid in selected_nodes
+ }
+ filtered_dependencies = {k: v for k, v in filtered_dependencies.items() if v}
+ return filtered_jobs, filtered_dependencies
+
+
+def filter_tasks_only(
+ full_jobs: dict, dependencies: dict[str, list[str]]
+) -> tuple[dict, dict[str, list[str]]]:
+ """Filter the graph to include only Task, Action, and LightweightTask (init_task) nodes."""
+ is_task = {
+ jid: isinstance(c, (Task, Action, LightweightTask)) or is_init_task_node(c)
+ for jid, c in full_jobs.items()
+ }
+ task_deps: dict[str, list[str]] = {}
+ for jid in full_jobs:
+ if not is_task[jid]:
+ continue
+ tdeps, q, processed = set(), list(dependencies.get(jid, [])), set()
+ while q:
+ d = q.pop()
+ if d in processed:
+ continue
+ processed.add(d)
+ if is_task.get(d):
+ tdeps.add(d)
+ else:
+ q.extend(dependencies.get(d, []))
+ task_deps[jid] = list(tdeps)
+ filtered_jobs = {jid: c for jid, c in full_jobs.items() if is_task[jid]}
+ return filtered_jobs, task_deps
+
+
+def _build_reduced_dag(
+ full_jobs: dict,
+ dependencies: dict[str, list[str]],
+) -> nx.DiGraph:
+ """Build a directed graph from jobs and dependencies, applying transitive reduction if acyclic."""
+ dag = nx.DiGraph()
+ for jid in full_jobs:
+ dag.add_node(jid)
+ for jid, deps in dependencies.items():
+ if jid in dag:
+ for d in deps:
+ if d in dag and d != jid:
+ dag.add_edge(d, jid)
+ return nx.transitive_reduction(dag) if nx.is_directed_acyclic_graph(dag) else dag
+
+
+def _infer_bottleneck_grid_keys(
+ full_jobs: dict,
+ job_tags_dict: dict[str, dict],
+) -> set[str] | None:
+ """Infer distinguishing bottleneck grid keys if multiple schemas are present."""
+ mock_records = [
+ {
+ "job_id": jid,
+ "task_id": type(c).__name__,
+ "tags": job_tags_dict.get(jid) or {},
+ }
+ for jid, c in full_jobs.items()
+ if isinstance(c, Task) and job_tags_dict.get(jid)
+ ]
+ detected = _detect_bottleneck_configs(mock_records)
+ if detected:
+ all_keys = set().union(*(r["tags"].keys() for r in mock_records))
+ det_keys = set().union(*(c.keys() for c in detected))
+ if det_keys < all_keys:
+ return det_keys
+ return None
+
+
+def _compute_node_base_signature(
+ config: Any,
+ tags: dict,
+ grid_keys: set[str] | None,
+) -> tuple:
+ """Compute base structural signature for a single job config."""
+ cname = type(config).__name__.replace(".XPMConfig", "")
+ ntype = (
+ "action"
+ if isinstance(config, Action)
+ else "task"
+ if isinstance(config, Task)
+ else "init"
+ if is_init_task_node(config)
+ else "config"
+ )
+ if grid_keys:
+ gtag_sig = tuple(sorted((k, str(v)) for k, v in tags.items() if k in grid_keys))
+ else:
+ gtag_sig = ()
+ return (cname, ntype, gtag_sig)
+
+
+def _refine_bisimulation_colors(
+ full_jobs: dict,
+ reduced_dag: nx.DiGraph,
+ base_sig: dict[str, tuple],
+) -> dict[str, int]:
+ """Iterative Weisfeiler-Lehman refinement on (base_sig, set(parent_colors), set(child_colors))."""
+ uniq_init = {v: i for i, v in enumerate(sorted(set(base_sig.values()), key=str))}
+ color: dict[str, int] = {jid: uniq_init[base_sig[jid]] for jid in full_jobs}
+
+ for _ in range(len(full_jobs) + 1):
+ new_sig: dict[str, tuple] = {}
+ for jid in full_jobs:
+ p_cols = tuple(sorted({color[p] for p in reduced_dag.predecessors(jid)}))
+ c_cols = tuple(sorted({color[c] for c in reduced_dag.successors(jid)}))
+ new_sig[jid] = (base_sig[jid], p_cols, c_cols)
+ uniq_map = {v: i for i, v in enumerate(sorted(set(new_sig.values()), key=str))}
+ new_color = {jid: uniq_map[new_sig[jid]] for jid in full_jobs}
+ if new_color == color:
+ break
+ color = new_color
+ return color
+
+
+def _merge_member_tags(
+ members: list[str],
+ job_tags_dict: dict[str, dict],
+) -> dict[str, Any]:
+ """Merge tags across stacked member nodes, noting multi-value variations."""
+ all_tkeys: list[str] = []
+ for m in members:
+ for k in (job_tags_dict.get(m) or {}).keys():
+ if k not in all_tkeys:
+ all_tkeys.append(k)
+ merged_tags: dict[str, Any] = {}
+ for k in all_tkeys:
+ vals = [
+ (job_tags_dict.get(m) or {}).get(k)
+ for m in members
+ if k in (job_tags_dict.get(m) or {})
+ ]
+ uniq_vals = {str(v) for v in vals}
+ if len(uniq_vals) == 1 and len(vals) == len(members):
+ merged_tags[k] = vals[0]
+ else:
+ merged_tags[k] = f"<{len(uniq_vals)} values>"
+ return merged_tags
+
+
+def _build_stacked_node_groups(
+ groups: dict[int, list[str]],
+ full_jobs: dict,
+ job_tags_dict: dict[str, dict],
+) -> tuple[dict[str, Any], dict[str, list[str]], dict[str, dict], dict[str, str]]:
+ """Build representative nodes, member mappings, and merged tags for each color group."""
+ member_to_rep: dict[str, str] = {}
+ stacked_jobs: dict[str, Any] = {}
+ stack_members: dict[str, list[str]] = {}
+ stacked_tags_dict: dict[str, dict] = {}
+
+ for members in groups.values():
+ rep_id = members[0]
+ stacked_jobs[rep_id] = full_jobs[rep_id]
+ stack_members[rep_id] = members
+ for m in members:
+ member_to_rep[m] = rep_id
+
+ if len(members) == 1:
+ if rep_id in job_tags_dict:
+ stacked_tags_dict[rep_id] = dict(job_tags_dict[rep_id])
+ else:
+ merged = _merge_member_tags(members, job_tags_dict)
+ if merged:
+ stacked_tags_dict[rep_id] = merged
+
+ return stacked_jobs, stack_members, stacked_tags_dict, member_to_rep
+
+
+def _remap_stacked_dependencies(
+ stack_members: dict[str, list[str]],
+ dependencies: dict[str, list[str]],
+ member_to_rep: dict[str, str],
+) -> dict[str, list[str]]:
+ """Remap dependencies using the representative node IDs and eliminate duplicates."""
+ stacked_dependencies: dict[str, list[str]] = {}
+ for rep_id, members in stack_members.items():
+ rep_deps: list[str] = []
+ seen_deps: set[str] = {rep_id}
+ for m in members:
+ for d in dependencies.get(m, []):
+ if d in member_to_rep:
+ d_rep = member_to_rep[d]
+ if d_rep not in seen_deps:
+ seen_deps.add(d_rep)
+ rep_deps.append(d_rep)
+ if rep_deps:
+ stacked_dependencies[rep_id] = rep_deps
+ return stacked_dependencies
+
+
+def stack_dag_nodes(
+ full_jobs: dict,
+ dependencies: dict[str, list[str]],
+ job_tags_dict: dict[str, dict],
+ grid_keys: set[str] | None = None,
+) -> tuple[dict, dict[str, list[str]], dict[str, dict], dict[str, list[str]]]:
+ """Stack structurally equivalent nodes sharing class name, grid tags, and parent/child groups.
+
+ Uses iterative bisimulation refinement on the transitively reduced DAG so that
+ parallel chains (e.g. per-dataset index -> 1st-stage evaluate -> 2nd-stage evaluate)
+ collapse into stacked nodes while keeping distinct core grid-search branches separate.
+
+ Returns:
+ Tuple of ``(stacked_jobs, stacked_dependencies, stacked_tags_dict, stack_members)``.
+ """
+ if not full_jobs:
+ return {}, {}, {}, {}
+
+ reduced_dag = _build_reduced_dag(full_jobs, dependencies)
+
+ if grid_keys is None:
+ grid_keys = _infer_bottleneck_grid_keys(full_jobs, job_tags_dict)
+
+ base_sig = {
+ jid: _compute_node_base_signature(
+ config, job_tags_dict.get(jid) or {}, grid_keys
+ )
+ for jid, config in full_jobs.items()
+ }
+ color = _refine_bisimulation_colors(full_jobs, reduced_dag, base_sig)
+
+ groups: dict[int, list[str]] = {}
+ for jid in sorted(full_jobs.keys()):
+ groups.setdefault(color[jid], []).append(jid)
+
+ stacked_jobs, stack_members, stacked_tags_dict, member_to_rep = (
+ _build_stacked_node_groups(groups, full_jobs, job_tags_dict)
+ )
+ stacked_dependencies = _remap_stacked_dependencies(
+ stack_members, dependencies, member_to_rep
+ )
+
+ return stacked_jobs, stacked_dependencies, stacked_tags_dict, stack_members
+
+
+def compute_dag_depths(
+ dag: nx.DiGraph,
+ full_jobs: dict,
+ align_types: bool = True,
+ aligned_class_names: set[str] | None = None,
+) -> dict[str, int]:
+ """Compute horizontal depths for nodes in a DAG with robust stage alignment.
+
+ Implements a 3-pass algorithm:
+ 1. Base Forward Topological Leveling:
+ depth(v) = max_{u in deps(v)}(depth(u) + 1), roots at 0.
+ 2. Grid Stage Alignment:
+ Aligns tasks of each type in aligned_class_names to max depth across peers.
+ 3. Deep Dependency Forward Push:
+ Topological push sweep ensuring depth(succ) >= depth(node) + 1 for all edges.
+ """
+ if aligned_class_names is None:
+ aligned_class_names = {"Learn", "ParameterLoader", "Evaluate"}
+
+ # Pass 1: Base Forward Topological Leveling
+ depths: dict[str, int] = {}
+ topo_order = list(nx.topological_sort(dag))
+ for node in topo_order:
+ preds = list(dag.predecessors(node))
+ if not preds:
+ depths[node] = 0
+ else:
+ depths[node] = max(depths[p] for p in preds) + 1
+
+ # Pass 2: Grid Stage Alignment
+ if align_types:
+ for tname in aligned_class_names:
+ nodes_t = [
+ n
+ for n in dag.nodes
+ if n in full_jobs and type(full_jobs[n]).__name__.split(".")[0] == tname
+ ]
+ if nodes_t:
+ max_d = max(depths[n] for n in nodes_t)
+ for n in nodes_t:
+ depths[n] = max_d
+
+ # Pass 3: Deep Dependency Forward Push
+ for node in topo_order:
+ for succ in dag.successors(node):
+ if depths[succ] <= depths[node]:
+ depths[succ] = depths[node] + 1
+
+ return depths
+
+
+def compute_dag_layout(
+ full_jobs: dict,
+ dependencies: dict[str, list[str]],
+ dependents: dict[str, list[str]],
+ align_types: bool = True,
+) -> tuple[dict[str, int], dict[str, tuple[float, float]], nx.DiGraph]:
+ """Compute depths and (X, Y) coordinates for the DAG to maximize straight arrows."""
+ dag = nx.DiGraph()
+ for jid in full_jobs:
+ dag.add_node(jid)
+ for u, targets in dependents.items():
+ for v in targets:
+ if u in dag and v in dag:
+ dag.add_edge(u, v)
+
+ depths = compute_dag_depths(dag, full_jobs, align_types=align_types)
+
+ reduced_dag = nx.transitive_reduction(dag)
+ preds: dict[str, list[str]] = {n: [] for n in full_jobs}
+ for u, v in reduced_dag.edges():
+ preds[v].append(u)
+
+ pos: dict[str, tuple[float, float]] = {}
+ unique_depths = sorted(set(depths.values()))
+ nodes_per_type: dict[str, list[str]] = {}
+ for d in unique_depths:
+ nodes_at_depth = [jid for jid, depth in depths.items() if depth == d]
+
+ # Compute desired Y (average of parents)
+ desired_y: dict[str, float] = {}
+ for jid in nodes_at_depth:
+ p_ys = [pos[p][1] for p in preds[jid] if p in pos]
+ tname = type(full_jobs[jid]).__name__.split(".")[0]
+ sibling_ys = [pos[s][1] for s in nodes_per_type.get(tname, []) if s in pos]
+ p_ys.extend(sibling_ys)
+ desired_y[jid] = sum(p_ys) / len(p_ys) if p_ys else 0.0
+
+ nodes_at_depth.sort(
+ key=lambda j: (-desired_y[j], type(full_jobs[j]).__name__, j)
+ )
+
+ for i, jid in enumerate(nodes_at_depth):
+ pos[jid] = (d * (BOX_W + MARGIN_X), -i * (BOX_H + MARGIN_Y))
+
+ for jid in nodes_at_depth:
+ tname = type(full_jobs[jid]).__name__.split(".")[0]
+ nodes_per_type.setdefault(tname, []).append(jid)
+
+ return depths, pos, reduced_dag
+
+
+def resolve_target_config(
+ args: argparse.Namespace,
+ configs: list[dict],
+) -> tuple[dict | None, str]:
+ """Resolve target configuration tags and title suffix from CLI options."""
+ if args.config_index is not None:
+ if not (0 <= args.config_index < len(configs)):
+ logging.error(
+ "Config index %d out of range [0..%d]. Use --list-configs to see options.",
+ args.config_index,
+ len(configs) - 1,
+ )
+ return None, ""
+ target_tags = configs[args.config_index]
+ return target_tags, ", ".join(
+ f"{k}={v}" for k, v in sorted(target_tags.items())
+ )
+
+ if args.filter:
+ filter_tags = parse_filter_arg(args.filter)
+ if args.single_config:
+ for cfg_cand in configs:
+ if all(cfg_cand.get(k) == v for k, v in filter_tags.items()):
+ return cfg_cand, ", ".join(
+ f"{k}={v}" for k, v in sorted(cfg_cand.items())
+ )
+ logging.error("No configuration matched filter: %s", args.filter)
+ return None, ""
+ return filter_tags, f"Filter: {args.filter}"
+
+ if args.single_config:
+ if not configs:
+ logging.error("No configurations found to filter.")
+ return None, ""
+ return configs[0], ", ".join(f"{k}={v}" for k, v in sorted(configs[0].items()))
+
+ return None, ""
+
+
+def build_dag_json(
+ full_jobs: dict,
+ dependencies: dict[str, list[str]],
+ job_tags_dict: dict[str, dict],
+ run_dir: Path | None = None,
+ experiment_id: str = "",
+ run_id: str = "",
+ workspace: str = "",
+) -> dict[str, Any]:
+ """Serialize the experiment DAG into a standardized, dual-compatible JSON format.
+
+ Supports both snake_case and camelCase field aliases so it can be consumed
+ transparently by draw_dag web renderers, experimaestro CLI, and web UI.
+ """
+ from experimaestro import Task, Action
+
+ # Read jobs metadata from jobs.jsonl and status.json if available
+ job_task_ids: dict[str, str] = {}
+ job_timestamps: dict[str, float] = {}
+ if run_dir:
+ jobs_jsonl = run_dir / "jobs.jsonl"
+ if jobs_jsonl.exists():
+ with jobs_jsonl.open() as f:
+ for line in f:
+ if line.strip():
+ try:
+ jd = json.loads(line)
+ jid = jd.get("job_id")
+ if jid:
+ if "task_id" in jd:
+ job_task_ids[jid] = jd["task_id"]
+ if "timestamp" in jd:
+ job_timestamps[jid] = jd["timestamp"]
+ except json.JSONDecodeError:
+ pass
+
+ workspace_val = str(workspace) if workspace else ""
+ if not workspace_val and run_dir and len(run_dir.parents) >= 3:
+ workspace_val = str(run_dir.parents[2])
+
+ status_data = load_run_status(run_dir) if run_dir else {}
+ job_states: dict[str, str] = status_data.get("job_states", {})
+
+ configs = get_experiment_configs(run_dir) if run_dir else []
+ grid_keys = set().union(*(c.keys() for c in configs)) if configs else None
+
+ # Compute deterministic layout matching flat plot
+ dependents: dict[str, list[str]] = {}
+ for jid, deps in dependencies.items():
+ for d in deps:
+ dependents.setdefault(d, []).append(jid)
+
+ depths, pos, reduced_dag = compute_dag_layout(full_jobs, dependencies, dependents)
+
+ nodes: list[dict[str, Any]] = []
+ for jid, config in full_jobs.items():
+ name = type(config).__name__.replace(".XPMConfig", "")
+ is_action = isinstance(config, Action)
+ is_task = isinstance(config, Task)
+ is_init = is_init_task_node(config)
+ ntype = (
+ "action"
+ if is_action
+ else "task"
+ if is_task
+ else "init"
+ if is_init
+ else "config"
+ )
+
+ # Determine task status
+ if jid in job_states:
+ status = job_states[jid]
+ elif is_init:
+ status = "init"
+ elif is_task or is_action:
+ status = "done" # If in finished run but not explicitly state-tracked
+ else:
+ status = "config"
+
+ # Tags
+ jtags = job_tags_dict.get(jid)
+ if not jtags and config:
+ try:
+ jtags = dict(tags(config))
+ except Exception:
+ jtags = {}
+ if not jtags and config:
+ jtags = extract_config_tags(config, grid_keys)
+ deps = dependencies.get(jid, [])
+
+ task_id = job_task_ids.get(jid, name)
+ if is_task or is_action:
+ if workspace_val:
+ full_path = str(Path(workspace_val) / "jobs" / task_id / jid)
+ else:
+ full_path = f"jobs/{task_id}/{jid}"
+ display_path = f".../{task_id}/{jid[:20]}..."
+ else:
+ full_path = ""
+ display_path = ""
+
+ coord = pos.get(jid, (0.0, 0.0))
+ depth_val = depths.get(jid, 0)
+ row_val = round(-coord[1] / (BOX_H + MARGIN_Y)) if (BOX_H + MARGIN_Y) > 0 else 0
+
+ node_dict = {
+ # Dual identifier keys
+ "id": jid,
+ "jobId": jid,
+ # Dual task naming keys
+ "name": name,
+ "task_id": task_id,
+ "taskId": task_id,
+ "type": ntype,
+ "status": status,
+ "init": is_init,
+ "tags": jtags,
+ # Dual dependency keys
+ "depends_on": deps,
+ "dependsOn": deps,
+ "path": full_path,
+ "display_path": display_path,
+ "locator": full_path,
+ "timestamp": job_timestamps.get(jid),
+ # Layout coordinates matching flat plot
+ "depth": depth_val,
+ "col": depth_val,
+ "row": row_val,
+ "x": coord[0],
+ "y": coord[1],
+ }
+ nodes.append(node_dict)
+
+ edges: list[dict[str, str]] = [
+ {"source": u, "target": v} for u, v in reduced_dag.edges()
+ ]
+
+ return {
+ "experiment_id": experiment_id,
+ "run_id": run_id,
+ "workspace": workspace_val,
+ "nodes": nodes,
+ "edges": edges,
+ "configs": configs,
+ "grid_keys": sorted(grid_keys) if grid_keys else [],
+ }
+
+
+def load_dag_from_json(path_or_str: str | Path) -> dict[str, Any]:
+ """Load an ExperimentGraph JSON from a file path or JSON string."""
+ p = Path(path_or_str) if isinstance(path_or_str, (str, Path)) else None
+ if p and p.exists():
+ with p.open() as f:
+ return json.load(f)
+ return json.loads(str(path_or_str))
diff --git a/src/experimaestro/tools/xpm_dag/xpm_web_dag.py b/src/experimaestro/tools/xpm_dag/xpm_web_dag.py
new file mode 100644
index 00000000..225a991a
--- /dev/null
+++ b/src/experimaestro/tools/xpm_dag/xpm_web_dag.py
@@ -0,0 +1,96 @@
+"""Interactive web DAG visualization generator and server for Experimaestro.
+
+This module generates a standalone, responsive, interactive HTML5 web page
+displaying the experiment DAG with client-side Dagre layout, D3 zoom/pan,
+custom card nodes, status coloring, Left-to-Right / Top-to-Bottom flow toggles,
+real-time single-config trial filtering, lineage path highlighting, and an
+integrated side inspector drawer.
+"""
+
+from __future__ import annotations
+
+import http.server
+from importlib.resources import files
+import json
+from pathlib import Path
+import socketserver
+import threading
+from typing import Any
+import webbrowser
+
+
+def _load_html_template() -> str:
+ """Load the standalone interactive HTML template."""
+ try:
+ return (
+ files("experimaestro.tools.xpm_dag")
+ .joinpath("dag_template.html")
+ .read_text(encoding="utf-8")
+ )
+ except (TypeError, ModuleNotFoundError, FileNotFoundError):
+ return (Path(__file__).parent / "dag_template.html").read_text(encoding="utf-8")
+
+
+HTML_TEMPLATE = _load_html_template()
+
+
+def generate_dag_html(
+ dag_json: dict[str, Any],
+ title: str = "",
+ default_layout: str = "dagre",
+) -> str:
+ """Generate a self-contained interactive HTML web page for the experiment DAG."""
+ exp_id = dag_json.get("experiment_id") or "Experiment"
+ run_id = dag_json.get("run_id") or ""
+ page_title = title or f"Experimaestro DAG - {exp_id}"
+
+ html = HTML_TEMPLATE.replace("__TITLE__", page_title)
+ html = html.replace("__RUN_ID__", run_id if run_id else "latest")
+ html = html.replace("__DEFAULT_LAYOUT__", default_layout)
+ html = html.replace("__GRAPH_DATA__", json.dumps(dag_json))
+ return html
+
+
+def serve_dag_web(
+ dag_json: dict[str, Any],
+ port: int = 0,
+ open_browser: bool = True,
+ default_layout: str = "dagre",
+) -> None:
+ """Start an ephemeral HTTP server to view the interactive DAG and open the browser."""
+ html_content = generate_dag_html(dag_json, default_layout=default_layout).encode(
+ "utf-8"
+ )
+
+ class DAGRequestHandler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self): # noqa: N802
+ self.send_response(200)
+ self.send_header("Content-type", "text/html; charset=utf-8")
+ self.send_header("Content-Length", str(len(html_content)))
+ self.end_headers()
+ self.wfile.write(html_content)
+
+ def log_message(self, format, *args): # noqa: A002
+ # Suppress normal HTTP request logging
+ pass
+
+ # Use ThreadingTCPServer with port 0 for automatic port selection
+ class ThreadedServer(socketserver.ThreadingTCPServer):
+ allow_reuse_address = True
+
+ server = ThreadedServer(("127.0.0.1", port), DAGRequestHandler)
+ actual_port = server.server_address[1]
+ url = f"http://localhost:{actual_port}"
+
+ print(f"Starting interactive DAG web server on {url}") # noqa: T201
+ print("Press Ctrl+C to stop the server.") # noqa: T201
+
+ if open_browser:
+ threading.Thread(target=lambda: webbrowser.open(url), daemon=True).start()
+
+ try:
+ server.serve_forever()
+ except KeyboardInterrupt:
+ print("\nShutting down web server...") # noqa: T201
+ finally:
+ server.server_close()
diff --git a/src/experimaestro/tui/__init__.py b/src/experimaestro/tui/__init__.py
index e7baf1bc..3a3fcd13 100644
--- a/src/experimaestro/tui/__init__.py
+++ b/src/experimaestro/tui/__init__.py
@@ -1,8 +1,11 @@
"""Textual-based TUI for monitoring experiments"""
+from pathlib import Path
from .app import ExperimaestroUI
# Backward compatibility alias
ExperimentTUI = ExperimaestroUI
-__all__ = ["ExperimaestroUI", "ExperimentTUI"]
+APP_TCSS_PATH = Path(__file__).parent / "app.tcss"
+
+__all__ = ["ExperimaestroUI", "ExperimentTUI", "APP_TCSS_PATH"]
diff --git a/src/experimaestro/tui/app.py b/src/experimaestro/tui/app.py
index 1896a7d5..26a05e0f 100644
--- a/src/experimaestro/tui/app.py
+++ b/src/experimaestro/tui/app.py
@@ -45,6 +45,8 @@
JobHighlighted,
ViewJobLogs,
ViewJobLogsRequest,
+ InspectJobParams,
+ InspectJobParamsRequest,
DeleteJobRequest,
DeleteExperimentRequest,
KillJobRequest,
@@ -87,6 +89,7 @@ class ExperimaestroUI(App):
Binding("?", "show_help", "Help"),
Binding("escape", "go_back", "Back", show=False),
Binding("l", "view_logs", "Logs", show=False),
+ Binding("i", "inspect_params", "Inspect Params", show=False),
Binding("w", "view_workspace_log", "Workspace Log", show=False),
Binding("j", "focus_jobs", "Jobs", show=False),
Binding("s", "focus_services", "Services", show=False),
@@ -702,6 +705,13 @@ def action_view_logs(self) -> None:
job_detail_view = self.query_one(JobDetailView)
job_detail_view.action_view_logs()
+ def action_inspect_params(self) -> None:
+ """Inspect parameters for the current job (if job detail is visible)"""
+ job_detail_container = self.query_one("#job-detail-container")
+ if not job_detail_container.has_class("hidden"):
+ job_detail_view = self.query_one(JobDetailView)
+ job_detail_view.action_inspect_params()
+
def action_view_workspace_log(self) -> None:
"""View workspace-level experimaestro log for current run"""
if not self.state_provider:
@@ -792,6 +802,43 @@ def on_view_job_logs_request(self, message: ViewJobLogsRequest) -> None:
return
self.post_message(ViewJobLogs(str(job.path), job.task_id, job.state))
+ def on_inspect_job_params_request(self, message: InspectJobParamsRequest) -> None:
+ """Handle params inspection request from jobs table or tree"""
+ job = self.state_provider.get_job(message.task_id, message.job_id)
+ if not job:
+ self.notify("Cannot find job", severity="warning")
+ return
+ self.post_message(InspectJobParams(job))
+
+ def on_inspect_job_params(self, message: InspectJobParams) -> None:
+ """Handle request to inspect job parameters - push ParamsInspectorScreen"""
+ from experimaestro.tools.inspect_params import ParamsInspectorScreen
+
+ job = message.job
+ if not self.state_provider.is_remote:
+ params_file = self.state_provider.get_job_params_file(job)
+ if not params_file or not params_file.is_file():
+ self.notify(
+ f"Parameters file not found for {job.identifier[:8]}",
+ severity="warning",
+ )
+ return
+ self.push_screen(
+ ParamsInspectorScreen(
+ params_target=params_file,
+ title=f"Job {job.identifier[:8]}",
+ )
+ )
+ else:
+ self.push_screen(
+ ParamsInspectorScreen(
+ params_target=None,
+ job=job,
+ state_provider=self.state_provider,
+ title=f"Job {job.identifier[:8]}",
+ )
+ )
+
def on_delete_job_request(self, message: DeleteJobRequest) -> None:
"""Handle job deletion request"""
job = self.state_provider.get_job(message.task_id, message.job_id)
diff --git a/src/experimaestro/tui/app.tcss b/src/experimaestro/tui/app.tcss
index d367c335..12e3f9cb 100644
--- a/src/experimaestro/tui/app.tcss
+++ b/src/experimaestro/tui/app.tcss
@@ -513,3 +513,95 @@ StrayJobsPanel Static,
OrphanJobsPanel Static {
height: auto;
}
+
+/* Parameters Inspector styles */
+ParamsInspectorScreen {
+ background: $surface;
+ color: $text;
+}
+
+#mode-bar {
+ height: auto;
+ background: $boost;
+ padding: 0 1;
+ border-bottom: solid $primary;
+}
+
+.title-badge {
+ padding: 0 1;
+ background: $primary;
+ color: $text;
+ text-style: bold;
+}
+
+#mode-hints {
+ color: $text-muted;
+ text-style: italic;
+ padding-left: 2;
+}
+
+#search-bar-container {
+ height: auto;
+ background: $boost;
+ padding: 0 1;
+ align: left middle;
+}
+
+#search-status {
+ padding: 0 1;
+ width: auto;
+ height: 100%;
+ content-align: center middle;
+}
+
+#search-status.error {
+ color: $error;
+ text-style: bold;
+}
+
+#search-status.valid {
+ color: $success;
+ text-style: bold;
+}
+
+#split-container {
+ width: 100%;
+ height: 1fr;
+}
+
+#tree-container {
+ width: 60%;
+ height: 100%;
+ border-right: solid $primary;
+}
+
+ParamsTree {
+ width: 100%;
+ height: 100%;
+ background: $surface;
+}
+
+#detail-pane {
+ width: 40%;
+ height: 100%;
+ padding: 1;
+ background: $surface;
+}
+
+#detail-content {
+ width: 100%;
+ height: auto;
+}
+
+.loading-message {
+ content-align: center middle;
+ height: 100%;
+ text-style: italic;
+ color: $text-muted;
+}
+
+.error-message {
+ content-align: center middle;
+ height: 100%;
+ color: $error;
+}
diff --git a/src/experimaestro/tui/dialogs.py b/src/experimaestro/tui/dialogs.py
index 2370eb66..b0f77ed5 100644
--- a/src/experimaestro/tui/dialogs.py
+++ b/src/experimaestro/tui/dialogs.py
@@ -137,6 +137,7 @@ def compose(self) -> ComposeResult:
[bold cyan]Jobs[/bold cyan]
l View job logs
+ i Inspect job parameters
Ctrl+d Delete job
Ctrl+k Kill running job
/ Open search filter
diff --git a/src/experimaestro/tui/messages.py b/src/experimaestro/tui/messages.py
index 4c4b26f9..8a51e8c9 100644
--- a/src/experimaestro/tui/messages.py
+++ b/src/experimaestro/tui/messages.py
@@ -4,7 +4,7 @@
from textual.message import Message
if TYPE_CHECKING:
- from experimaestro.scheduler.interfaces import JobState
+ from experimaestro.scheduler.interfaces import BaseJob, JobState
class ExperimentSelected(Message):
@@ -60,6 +60,24 @@ def __init__(self, job_id: str, task_id: str, experiment_id: str) -> None:
self.experiment_id = experiment_id
+class InspectJobParams(Message):
+ """Message sent when user wants to inspect job parameters"""
+
+ def __init__(self, job: "BaseJob") -> None:
+ super().__init__()
+ self.job = job
+
+
+class InspectJobParamsRequest(Message):
+ """Message sent when user requests to inspect parameters from jobs table or tree"""
+
+ def __init__(self, job_id: str, task_id: str, experiment_id: str) -> None:
+ super().__init__()
+ self.job_id = job_id
+ self.task_id = task_id
+ self.experiment_id = experiment_id
+
+
class DeleteJobRequest(Message):
"""Message sent when user requests to delete a job"""
diff --git a/src/experimaestro/tui/widgets/jobs.py b/src/experimaestro/tui/widgets/jobs.py
index a0342453..6ae248bd 100644
--- a/src/experimaestro/tui/widgets/jobs.py
+++ b/src/experimaestro/tui/widgets/jobs.py
@@ -25,6 +25,8 @@
JobHighlighted,
ViewJobLogs,
ViewJobLogsRequest,
+ InspectJobParams,
+ InspectJobParamsRequest,
DeleteJobRequest,
KillJobRequest,
FilterChanged,
@@ -238,6 +240,7 @@ class JobDetailView(Widget):
BINDINGS = [
Binding("l", "view_logs", "View Logs", priority=True),
+ Binding("i", "inspect_params", "Inspect Params", priority=True),
]
def __init__(self, state_provider: StateProvider) -> None:
@@ -287,6 +290,11 @@ def action_view_logs(self) -> None:
)
)
+ def action_inspect_params(self) -> None:
+ """Inspect job parameters with TUI inspector"""
+ if self.job_data:
+ self.post_message(InspectJobParams(self.job_data))
+
def _get_process_info(self, job) -> str:
"""Get process information for a job using the state provider"""
pinfo = self.state_provider.get_process_info(job)
@@ -555,21 +563,29 @@ def format_time(ts):
progress_text = "-"
self.query_one("#job-progress-label", Label).update(progress_text)
- # Log files hint - log files are named after the last part of the task ID
+ # Actions hint (logs and parameters)
job_path = job.path
task_id = job.task_id
if job_path and task_id:
- # Extract the last component of the task ID (e.g., "evaluate" from "mnist_xp.learn.evaluate")
- task_name = task_id.split(".")[-1]
- stdout_path = job_path / f"{task_name}.out"
- stderr_path = job_path / f"{task_name}.err"
- logs_exist = stdout_path.exists() or stderr_path.exists()
- if logs_exist:
- self.query_one("#job-logs-hint", Label).update(
- "[bold cyan]Press 'l' to view logs[/bold cyan]"
- )
+ if self.state_provider.is_remote:
+ logs_exist = bool(job.state and job.state != JobState.SCHEDULED)
+ params_exist = self.state_provider.has_job_params(job)
+ else:
+ task_name = task_id.split(".")[-1]
+ stdout_path = job_path / f"{task_name}.out"
+ stderr_path = job_path / f"{task_name}.err"
+ logs_exist = stdout_path.exists() or stderr_path.exists()
+ params_exist = self.state_provider.has_job_params(job)
+
+ if logs_exist and params_exist:
+ hint = "[bold cyan]Press 'l' to view logs, 'i' to inspect params[/bold cyan]"
+ elif params_exist:
+ hint = "[bold cyan]Press 'i' to inspect params[/bold cyan] [dim](no logs yet)[/dim]"
+ elif logs_exist:
+ hint = "[bold cyan]Press 'l' to view logs[/bold cyan]"
else:
- self.query_one("#job-logs-hint", Label).update("(no log files found)")
+ hint = "(no log or param files found)"
+ self.query_one("#job-logs-hint", Label).update(hint)
else:
self.query_one("#job-logs-hint", Label).update("")
@@ -624,6 +640,7 @@ class JobsTreeView(Widget):
Binding("E", "collapse_all", "Collapse All", show=False),
Binding("enter", "select_node", "Select", show=False),
Binding("l", "view_logs", "Logs", key_display="l"),
+ Binding("i", "inspect_params", "Inspect", key_display="i"),
Binding("ctrl+d", "delete_job", "Delete", show=False),
Binding("ctrl+k", "kill_job", "Kill", show=False),
]
@@ -1150,6 +1167,15 @@ def action_view_logs(self) -> None:
ViewJobLogsRequest(data.job_id, data.task_id, self.current_experiment)
)
+ def action_inspect_params(self) -> None:
+ data = self._get_selected_job_data()
+ if data and self.current_experiment:
+ self.post_message(
+ InspectJobParamsRequest(
+ data.job_id, data.task_id, self.current_experiment
+ )
+ )
+
def action_delete_job(self) -> None:
data = self._get_selected_job_data()
if data and self.current_experiment:
@@ -1180,6 +1206,7 @@ class JobsTable(Vertical):
Binding("ctrl+d", "delete_job", "Delete", show=False),
Binding("ctrl+k", "kill_job", "Kill", show=False),
Binding("l", "view_logs", "Logs", key_display="l"),
+ Binding("i", "inspect_params", "Inspect", key_display="i"),
Binding("f", "copy_path", "Copy Path", show=False),
Binding("/", "toggle_search", "Search"),
Binding("c", "clear_filter", "Clear", show=False),
@@ -1414,6 +1441,18 @@ def action_view_logs(self) -> None:
ViewJobLogsRequest(job_id, task_id, self.current_experiment)
)
+ def action_inspect_params(self) -> None:
+ """Request to inspect params for the selected job"""
+ selected = self._get_selected_job_info()
+ if selected is None:
+ return
+
+ job_id, task_id = selected
+ if task_id:
+ self.post_message(
+ InspectJobParamsRequest(job_id, task_id, self.current_experiment)
+ )
+
def _get_selected_job_info(self) -> tuple[str, str] | None:
"""Get (job_id, task_id) for the currently selected job in either view.
diff --git a/src/experimaestro/utils/remote.py b/src/experimaestro/utils/remote.py
new file mode 100644
index 00000000..00934dab
--- /dev/null
+++ b/src/experimaestro/utils/remote.py
@@ -0,0 +1,383 @@
+"""Remote workspace utilities via SSH and Rsync.
+
+Provides lightweight remote querying, pattern matching, and file synchronization
+to support tools like `experimaestro inspect` and `experimaestro dag` on remote SSH workspaces.
+"""
+
+from __future__ import annotations
+
+import atexit
+import logging
+from pathlib import Path
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Any, List, Optional, Tuple
+
+logger = logging.getLogger("experimaestro")
+
+_SESSION_REMOTE_CACHE_DIR: Optional[Path] = None
+
+
+def get_session_cache_dir() -> Path:
+ """Return a managed temporary directory for cached remote workspace files.
+
+ Creates the directory lazily on first access and registers automatic cleanup
+ upon process termination via atexit.
+
+ Returns:
+ Path to the session temporary cache directory.
+ """
+ global _SESSION_REMOTE_CACHE_DIR
+ if _SESSION_REMOTE_CACHE_DIR is None or not _SESSION_REMOTE_CACHE_DIR.exists():
+ tmp = tempfile.mkdtemp(prefix="xpm_cache_")
+ _SESSION_REMOTE_CACHE_DIR = Path(tmp)
+ atexit.register(shutil.rmtree, tmp, ignore_errors=True)
+ return _SESSION_REMOTE_CACHE_DIR
+
+
+def clean_job_target(
+ raw: str, ws_base: str = "", filename: Optional[str] = None
+) -> str:
+ """Normalize a target job locator, prefix, or path.
+
+ Strips surrounding slashes, explicit file names, workspace root prefixes,
+ and leading 'jobs/' prefixes.
+
+ Args:
+ raw: Raw input target string.
+ ws_base: Optional workspace base path to strip.
+ filename: Optional filename to strip from the end (e.g. 'params.json').
+
+ Returns:
+ Normalized target string (e.g. 'ee76fb8f' or 'my_task/ee76fb8f').
+
+ Raises:
+ ValueError: If the normalized target is empty or root.
+ """
+ clean = raw.strip("/")
+ if filename:
+ clean = clean.removesuffix(f"/{filename}").removesuffix(filename).strip("/")
+ else:
+ for ext in (".json", ".log", ".txt", ".out", ".err"):
+ if clean.endswith(ext):
+ clean = str(Path(clean).parent)
+ break
+
+ ws_clean = ws_base.strip("/")
+ if ws_clean and clean.startswith(ws_clean):
+ clean = clean[len(ws_clean) :].strip("/")
+
+ if "/jobs/" in clean:
+ clean = clean.split("/jobs/", 1)[1].strip("/")
+ elif clean.startswith("jobs/"):
+ clean = clean[len("jobs/") :].strip("/")
+
+ if not clean or clean == ".":
+ raise ValueError("Job target cannot be empty or root.")
+
+ return clean
+
+
+def resolve_job_file_pattern(clean_target: str, filename: str) -> str:
+ """Construct workspace-relative glob pattern for a target and filename.
+
+ Args:
+ clean_target: Normalized target (e.g. 'my_task/ee76fb8f' or 'ee76fb8f').
+ filename: Relative filename to find (e.g. 'params.json', 'experimaestro.json').
+
+ Returns:
+ Workspace-relative glob pattern string under the jobs directory.
+ """
+ clean_target = clean_target.replace(":", "/")
+ if "/" in clean_target:
+ return f"jobs/{clean_target}*/{filename}"
+ return f"jobs/*/{clean_target}*/{filename}"
+
+
+def build_ssh_cmd(ssh_settings: Any, remote_command: str) -> List[str]:
+ """Construct an SSH command list respecting options and shell initialization.
+
+ Args:
+ ssh_settings: SSHSettings object with host, options, and shell_init.
+ remote_command: Command string to execute on the remote host.
+
+ Returns:
+ Command list suitable for subprocess.run.
+ """
+ cmd = ["ssh"]
+ if ssh_settings.options:
+ cmd.extend(ssh_settings.options)
+ cmd.append(ssh_settings.host)
+
+ full_remote_cmd = remote_command
+ if ssh_settings.shell_init:
+ full_remote_cmd = f"{ssh_settings.shell_init}; {remote_command}"
+
+ cmd.append(full_remote_cmd)
+ return cmd
+
+
+def run_ssh_command(
+ ssh_settings: Any, remote_command: str
+) -> subprocess.CompletedProcess:
+ """Execute a command string on the remote host via SSH with captured output.
+
+ Args:
+ ssh_settings: SSHSettings instance.
+ remote_command: Command string to run remotely.
+
+ Returns:
+ CompletedProcess instance with stdout and stderr.
+ """
+ cmd = build_ssh_cmd(ssh_settings, remote_command)
+ return subprocess.run(cmd, capture_output=True, text=True)
+
+
+def fetch_remote_file_rsync(
+ ssh_settings: Any, remote_path: str, local_dest: Path
+) -> None:
+ """Transfer a single remote file to local_dest using rsync or scp.
+
+ Args:
+ ssh_settings: SSHSettings object with host and options.
+ remote_path: Absolute or workspace path on the remote host.
+ local_dest: Local Path where the file should be written.
+
+ Raises:
+ RuntimeError: If both rsync and scp transfer attempts fail.
+ """
+ local_dest.parent.mkdir(parents=True, exist_ok=True)
+ source = f"{ssh_settings.host}:{remote_path}"
+ dest = str(local_dest)
+
+ cmd = ["rsync", "-L", "-a"]
+ if ssh_settings.options:
+ ssh_opt_str = " ".join(shlex.quote(opt) for opt in ssh_settings.options)
+ cmd.extend(["-e", f"ssh {ssh_opt_str}"])
+ cmd.extend([source, dest])
+
+ res = subprocess.run(cmd, capture_output=True, text=True)
+ if res.returncode != 0:
+ # Fallback to scp if rsync failed or is unavailable
+ scp_cmd = ["scp"]
+ if ssh_settings.options:
+ scp_cmd.extend(ssh_settings.options)
+ scp_cmd.extend([source, dest])
+ scp_res = subprocess.run(scp_cmd, capture_output=True, text=True)
+ if scp_res.returncode != 0:
+ err = res.stderr.strip() or scp_res.stderr.strip()
+ raise RuntimeError(
+ f"Failed to transfer {remote_path} from {ssh_settings.host}: {err}"
+ )
+
+
+def fetch_remote_files_rsync(
+ ssh_settings: Any,
+ remote_src_dir: str,
+ local_dest_dir: Path,
+ includes: List[str],
+) -> None:
+ """Transfer specific files matching includes from remote directory using rsync.
+
+ Args:
+ ssh_settings: SSHSettings object.
+ remote_src_dir: Remote source directory path.
+ local_dest_dir: Local destination directory.
+ includes: List of filename patterns to include (e.g. ['objects.jsonl', 'jobs.jsonl']).
+ """
+ local_dest_dir.mkdir(parents=True, exist_ok=True)
+ source = f"{ssh_settings.host}:{remote_src_dir.rstrip('/')}/"
+ dest = f"{str(local_dest_dir).rstrip('/')}/"
+
+ cmd = ["rsync", "-L", "-a"]
+ if ssh_settings.options:
+ ssh_opt_str = " ".join(shlex.quote(opt) for opt in ssh_settings.options)
+ cmd.extend(["-e", f"ssh {ssh_opt_str}"])
+
+ for inc in includes:
+ cmd.extend(["--include", inc])
+ cmd.extend(["--exclude", "*", source, dest])
+
+ res = subprocess.run(cmd, capture_output=True, text=True)
+ if res.returncode != 0:
+ # Fallback to pulling files individually
+ for inc in includes:
+ remote_file = f"{remote_src_dir.rstrip('/')}/{inc}"
+ local_file = local_dest_dir / inc
+ try:
+ fetch_remote_file_rsync(ssh_settings, remote_file, local_file)
+ except Exception:
+ pass
+
+
+def find_remote_workspace_file(
+ ws: Any,
+ target: str,
+ filename: str = "params.json",
+ cache_dir: Optional[Path] = None,
+) -> Path:
+ """Fetch a file from a remote workspace job directory using SSH and Rsync.
+
+ Resolves remote workspace job directories matching `target` and `filename`
+ via SSH pattern matching, and synchronizes the file into a local cache directory.
+
+ Args:
+ ws: Remote WorkspaceSettings instance with SSH configuration.
+ target: Job ID / hash prefix or remote job directory subpath.
+ filename: Relative filename to fetch (default: 'params.json').
+ cache_dir: Optional local directory to store fetched files. If None,
+ uses the managed session temporary cache directory.
+
+ Returns:
+ Local Path to the synchronized file.
+
+ Raises:
+ FileNotFoundError: If target is empty, or no matching file exists remotely.
+ RuntimeError: If remote SSH command execution or file transfer fails.
+ """
+ ws_raw = str(getattr(ws, "_raw_path", "")).strip("/")
+ try:
+ clean_target = clean_job_target(target, ws_raw, filename=filename)
+ except ValueError as e:
+ raise FileNotFoundError(
+ f"Please specify a job ID or path for remote workspace '{ws.id}'."
+ ) from e
+
+ pattern = resolve_job_file_pattern(clean_target, filename)
+ remote_pattern = f"{ws._raw_path}/{pattern}"
+
+ logger.info(
+ "Querying remote workspace %s via SSH for %s matching '%s'...",
+ ws.id,
+ filename,
+ clean_target,
+ )
+
+ py_bin = ws.ssh.python or "python3"
+ py_script = "import glob, sys; [print(p) for p in glob.glob(sys.argv[1])]"
+ remote_cmd = f"{py_bin} -c {py_script!r} {shlex.quote(remote_pattern)}"
+
+ res = run_ssh_command(ws.ssh, remote_cmd)
+ if res.returncode != 0:
+ err = res.stderr.strip() or res.stdout.strip()
+ raise RuntimeError(f"Failed to query remote workspace '{ws.id}' via SSH: {err}")
+
+ matches = [line.strip() for line in res.stdout.splitlines() if line.strip()]
+ if not matches:
+ raise FileNotFoundError(
+ f"No job matching '{clean_target}' found in remote workspace '{ws.id}' ({ws._raw_path})"
+ )
+
+ if len(matches) > 1:
+ print( # noqa: T201
+ f"Warning: Multiple jobs ({len(matches)}) matched '{clean_target}'. "
+ f"Using the first match: {matches[0]}",
+ file=sys.stderr,
+ )
+
+ remote_file_path = matches[0]
+ rel_path = remote_file_path[len(ws._raw_path) :].lstrip("/")
+ dest_cache = cache_dir or get_session_cache_dir()
+ local_dest = dest_cache / rel_path
+
+ logger.info("Synchronizing %s from %s into %s...", rel_path, ws.id, local_dest)
+ fetch_remote_file_rsync(ws.ssh, remote_file_path, local_dest)
+
+ if not local_dest.exists():
+ raise FileNotFoundError(
+ f"{filename} not found in local cache {local_dest} after transfer."
+ )
+
+ return local_dest
+
+
+fetch_remote_workspace_file = find_remote_workspace_file
+
+
+def find_remote_workspace_experiment_run(
+ ws: Any,
+ experiment_id: str,
+ run_id: Optional[str] = None,
+ cache_dir: Optional[Path] = None,
+) -> Tuple[str, Path]:
+ """Find and synchronize an experiment run's metadata from a remote workspace.
+
+ Queries the remote workspace for available runs in `experiments//`,
+ selects the requested run (or the most recent one), and pulls `objects.jsonl`,
+ `jobs.jsonl`, and `progress.jsonl` into the local session cache directory.
+
+ Args:
+ ws: Remote WorkspaceSettings instance.
+ experiment_id: Experiment identifier.
+ run_id: Optional specific run timestamp ID. If None, picks the latest.
+ cache_dir: Optional cache directory.
+
+ Returns:
+ Tuple of (selected_run_id, local_run_dir_path).
+
+ Raises:
+ FileNotFoundError: If experiment or run cannot be found remotely.
+ RuntimeError: If remote SSH command execution fails.
+ """
+ ws_raw = str(getattr(ws, "_raw_path", "")).rstrip("/")
+ exp_remote_path = f"{ws_raw}/experiments/{experiment_id}"
+
+ py_bin = ws.ssh.python or "python3"
+ # List run directories under the experiment directory
+ py_script = (
+ "import os, sys, glob; "
+ "d = sys.argv[1]; "
+ "runs = [os.path.basename(p) for p in glob.glob(os.path.join(d, '*')) if os.path.isdir(p)]; "
+ "print('\\n'.join(sorted(runs)))"
+ )
+ remote_cmd = f"{py_bin} -c {py_script!r} {shlex.quote(exp_remote_path)}"
+
+ res = run_ssh_command(ws.ssh, remote_cmd)
+ if res.returncode != 0:
+ err = res.stderr.strip() or res.stdout.strip()
+ raise RuntimeError(
+ f"Failed to query experiment '{experiment_id}' in remote workspace '{ws.id}': {err}"
+ )
+
+ available_runs = [line.strip() for line in res.stdout.splitlines() if line.strip()]
+ if not available_runs:
+ raise FileNotFoundError(
+ f"No runs found for experiment '{experiment_id}' in remote workspace '{ws.id}'."
+ )
+
+ if run_id:
+ if run_id not in available_runs:
+ raise FileNotFoundError(
+ f"Run '{run_id}' not found for experiment '{experiment_id}' in remote workspace '{ws.id}'. "
+ f"Available runs: {available_runs}"
+ )
+ selected_run_id = run_id
+ else:
+ selected_run_id = available_runs[-1]
+
+ remote_run_dir = f"{exp_remote_path}/{selected_run_id}"
+ dest_cache = cache_dir or get_session_cache_dir()
+ local_run_dir = dest_cache / "experiments" / experiment_id / selected_run_id
+ local_run_dir.mkdir(parents=True, exist_ok=True)
+
+ logger.info(
+ "Synchronizing DAG files for run %s (%s) from %s into %s...",
+ selected_run_id,
+ experiment_id,
+ ws.id,
+ local_run_dir,
+ )
+
+ # Fetch lightweight metadata files needed for DAG visualization
+ includes = ["objects.jsonl", "jobs.jsonl", "progress.jsonl", "run.yaml"]
+ fetch_remote_files_rsync(ws.ssh, remote_run_dir, local_run_dir, includes)
+
+ if not (local_run_dir / "objects.jsonl").exists():
+ # Fallback: pull objects.jsonl individually to verify
+ remote_obj = f"{remote_run_dir}/objects.jsonl"
+ fetch_remote_file_rsync(ws.ssh, remote_obj, local_run_dir / "objects.jsonl")
+
+ return selected_run_id, local_run_dir
diff --git a/uv.lock b/uv.lock
index f67c0513..5811ab6f 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
version = 1
-revision = 5
+revision = 3
requires-python = ">=3.11"
resolution-markers = [
"python_full_version >= '3.15' and sys_platform == 'win32'",
@@ -720,6 +720,8 @@ dependencies = [
{ name = "humanfriendly" },
{ name = "jinja2" },
{ name = "marshmallow" },
+ { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
+ { name = "networkx", version = "3.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "psutil" },
{ name = "pydantic" },
{ name = "pyparsing" },
@@ -798,6 +800,7 @@ requires-dist = [
{ name = "marshmallow", specifier = ">=3.20,<4" },
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0" },
{ name = "myst-parser", marker = "extra == 'docs'", specifier = ">=2.0" },
+ { name = "networkx", specifier = ">=3.6.1" },
{ name = "paramiko", marker = "extra == 'ssh'", specifier = ">=3.3" },
{ name = "psutil", specifier = ">=7,<8" },
{ name = "pydantic", specifier = ">=2.6" },
@@ -1867,6 +1870,40 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl", hash = "sha256:60567d774edf77db53906f89d9fbd164e66e56d66d388e1e6990f17ac33cfb53", size = 454815, upload-time = "2026-06-05T12:34:32.289Z" },
]
+[[package]]
+name = "networkx"
+version = "3.6.1"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+ "python_full_version < '3.12' and sys_platform == 'win32'",
+ "python_full_version < '3.12' and sys_platform == 'emscripten'",
+ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" },
+]
+
+[[package]]
+name = "networkx"
+version = "3.7"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+ "python_full_version >= '3.15' and sys_platform == 'win32'",
+ "python_full_version >= '3.15' and sys_platform == 'emscripten'",
+ "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+ "python_full_version == '3.14.*' and sys_platform == 'win32'",
+ "python_full_version == '3.14.*' and sys_platform == 'emscripten'",
+ "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'",
+ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'",
+ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz", hash = "sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", size = 1866482, upload-time = "2026-09-21T16:45:16.974Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl", hash = "sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", size = 2142205, upload-time = "2026-09-21T16:45:14.609Z" },
+]
+
[[package]]
name = "numpy"
version = "2.4.6"
@@ -2480,7 +2517,7 @@ resolution-markers = [
"python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
- { name = "typing-extensions" },
+ { name = "typing-extensions", marker = "python_full_version < '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
wheels = [
@@ -2503,7 +2540,7 @@ resolution-markers = [
"python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
- { name = "typing-extensions" },
+ { name = "typing-extensions", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/fdedc2c75c3e31a330659c85e5793bb18b3397981fbf0844c6dee5b18926/pyee-14.0.0.tar.gz", hash = "sha256:76dd0f4314ecd27f02dc73589dea7fd3853f9b6176d8ef9b122860657e3602de", size = 98760, upload-time = "2026-08-13T04:26:11.021Z" }
wheels = [
@@ -2887,13 +2924,13 @@ name = "scikit-learn"
version = "1.9.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "joblib" },
- { name = "narwhals" },
- { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
- { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
- { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
- { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
- { name = "threadpoolctl" },
+ { name = "joblib", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "narwhals", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "threadpoolctl", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" }
wheels = [
@@ -2917,7 +2954,7 @@ resolution-markers = [
"python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
- { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } },
+ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
wheels = [
@@ -2957,7 +2994,7 @@ resolution-markers = [
"python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
- { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" } },
+ { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" }
wheels = [
@@ -3043,23 +3080,23 @@ resolution-markers = [
"python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
- { name = "alabaster" },
- { name = "babel" },
- { name = "colorama", marker = "sys_platform == 'win32'" },
- { name = "docutils" },
- { name = "imagesize" },
- { name = "jinja2" },
- { name = "packaging" },
- { name = "pygments" },
- { name = "requests" },
- { name = "roman-numerals" },
- { name = "snowballstemmer" },
- { name = "sphinxcontrib-applehelp" },
- { name = "sphinxcontrib-devhelp" },
- { name = "sphinxcontrib-htmlhelp" },
- { name = "sphinxcontrib-jsmath" },
- { name = "sphinxcontrib-qthelp" },
- { name = "sphinxcontrib-serializinghtml" },
+ { name = "alabaster", marker = "python_full_version < '3.12'" },
+ { name = "babel", marker = "python_full_version < '3.12'" },
+ { name = "colorama", marker = "python_full_version < '3.12' and sys_platform == 'win32'" },
+ { name = "docutils", marker = "python_full_version < '3.12'" },
+ { name = "imagesize", marker = "python_full_version < '3.12'" },
+ { name = "jinja2", marker = "python_full_version < '3.12'" },
+ { name = "packaging", marker = "python_full_version < '3.12'" },
+ { name = "pygments", marker = "python_full_version < '3.12'" },
+ { name = "requests", marker = "python_full_version < '3.12'" },
+ { name = "roman-numerals", marker = "python_full_version < '3.12'" },
+ { name = "snowballstemmer", marker = "python_full_version < '3.12'" },
+ { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.12'" },
+ { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.12'" },
+ { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.12'" },
+ { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.12'" },
+ { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.12'" },
+ { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" }
wheels = [
@@ -3082,23 +3119,23 @@ resolution-markers = [
"python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
- { name = "alabaster" },
- { name = "babel" },
- { name = "colorama", marker = "sys_platform == 'win32'" },
- { name = "docutils" },
- { name = "imagesize" },
- { name = "jinja2" },
- { name = "packaging" },
- { name = "pygments" },
- { name = "requests" },
- { name = "roman-numerals" },
- { name = "snowballstemmer" },
- { name = "sphinxcontrib-applehelp" },
- { name = "sphinxcontrib-devhelp" },
- { name = "sphinxcontrib-htmlhelp" },
- { name = "sphinxcontrib-jsmath" },
- { name = "sphinxcontrib-qthelp" },
- { name = "sphinxcontrib-serializinghtml" },
+ { name = "alabaster", marker = "python_full_version >= '3.12'" },
+ { name = "babel", marker = "python_full_version >= '3.12'" },
+ { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" },
+ { name = "docutils", marker = "python_full_version >= '3.12'" },
+ { name = "imagesize", marker = "python_full_version >= '3.12'" },
+ { name = "jinja2", marker = "python_full_version >= '3.12'" },
+ { name = "packaging", marker = "python_full_version >= '3.12'" },
+ { name = "pygments", marker = "python_full_version >= '3.12'" },
+ { name = "requests", marker = "python_full_version >= '3.12'" },
+ { name = "roman-numerals", marker = "python_full_version >= '3.12'" },
+ { name = "snowballstemmer", marker = "python_full_version >= '3.12'" },
+ { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" },
+ { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" },
+ { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" },
+ { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" },
+ { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" },
+ { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" }
wheels = [
@@ -3324,7 +3361,7 @@ name = "typeguard"
version = "4.5.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "typing-extensions" },
+ { name = "typing-extensions", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/67/1c/dfba5c4633cafc4c701f237d2ba63b416805047fd6d96aab4cfc40969f98/typeguard-4.5.2.tar.gz", hash = "sha256:5a16dcac23502039299c97c8941651bc33d7ea8cc4b2f7d6bbb1b528f6eea423", size = 80240, upload-time = "2026-05-14T12:59:40.857Z" }
wheels = [
@@ -3372,9 +3409,9 @@ name = "tyro"
version = "1.0.15"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "docstring-parser" },
- { name = "typeguard" },
- { name = "typing-extensions" },
+ { name = "docstring-parser", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "typeguard", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "typing-extensions", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/12/78/a5749a6c1ee9abc2999e294f339f8f72476d1a60bb95fc0e86156aafed3b/tyro-1.0.15.tar.gz", hash = "sha256:3f1d60887723eecb9c489f195d11f079c4a1f33df74b723552ad31ec57c667bb", size = 593822, upload-time = "2026-06-20T08:48:28.364Z" }
wheels = [
@@ -3857,17 +3894,17 @@ name = "zeus"
version = "0.15.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "amdsmi" },
- { name = "httpx" },
- { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
- { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
- { name = "nvidia-ml-py" },
- { name = "pandas" },
- { name = "pydantic" },
- { name = "python-dateutil" },
- { name = "rich" },
- { name = "scikit-learn" },
- { name = "tyro" },
+ { name = "amdsmi", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "httpx", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "nvidia-ml-py", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "pandas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "pydantic", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "python-dateutil", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "rich", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "scikit-learn", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
+ { name = "tyro", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e9/a1/29f935d77a45cf5174db3c42f9f4ab439bb03d13f69a7748ec95d3a94509/zeus-0.15.1.tar.gz", hash = "sha256:f7b347a93dc729c9dfa2379bbfe25c45fe58e659223a0930e76701b1441c5ca9", size = 227120, upload-time = "2026-03-29T05:44:14.433Z" }
wheels = [
@@ -3876,7 +3913,7 @@ wheels = [
[package.optional-dependencies]
apple = [
- { name = "zeus-apple-silicon" },
+ { name = "zeus-apple-silicon", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
]
[[package]]