From 87af6fc051d1e8af8dfd4ee7202f87b320ef127b Mon Sep 17 00:00:00 2001 From: Victor Morand Date: Mon, 19 Oct 2026 10:28:38 +0200 Subject: [PATCH 1/7] feat: add a DAG drawing tool --- src/experimaestro/tests/test_xpm_dag.py | 212 +++ src/experimaestro/tools/xpm_dag/__init__.py | 57 + src/experimaestro/tools/xpm_dag/__main__.py | 6 + .../tools/xpm_dag/dag_template.html | 1501 +++++++++++++++++ src/experimaestro/tools/xpm_dag/draw_dag.py | 591 +++++++ src/experimaestro/tools/xpm_dag/xpm_dag.py | 663 ++++++++ .../tools/xpm_dag/xpm_web_dag.py | 96 ++ 7 files changed, 3126 insertions(+) create mode 100644 src/experimaestro/tests/test_xpm_dag.py create mode 100644 src/experimaestro/tools/xpm_dag/__init__.py create mode 100644 src/experimaestro/tools/xpm_dag/__main__.py create mode 100644 src/experimaestro/tools/xpm_dag/dag_template.html create mode 100644 src/experimaestro/tools/xpm_dag/draw_dag.py create mode 100644 src/experimaestro/tools/xpm_dag/xpm_dag.py create mode 100644 src/experimaestro/tools/xpm_dag/xpm_web_dag.py diff --git a/src/experimaestro/tests/test_xpm_dag.py b/src/experimaestro/tests/test_xpm_dag.py new file mode 100644 index 00000000..7b9f9846 --- /dev/null +++ b/src/experimaestro/tests/test_xpm_dag.py @@ -0,0 +1,212 @@ +"""Tests for the standalone experimaestro DAG visualization tool.""" + +import json +import logging + +from experimaestro import Config, Param, Task +from experimaestro.tools.xpm_dag import ( + build_dag_json, + draw_dag, + extract_config_tags, + generate_dag_html, + get_experiment_configs, +) + + +class DummyModel(Config): + hidden_dim: Param[int] + kernel_size: Param[int] + n_layers: Param[int] + + +class DummyTask(Task): + pass + + +def test_extract_config_tags(): + """Verify extract_config_tags extracts hyperparameter tags from Config objects.""" + model = DummyModel.C(hidden_dim=32, kernel_size=3, n_layers=2) + + # 1. With grid_keys: filters to specified grid dimensions + tags_filtered = extract_config_tags(model, {"hidden_dim", "kernel_size"}) + assert tags_filtered == {"hidden_dim": 32, "kernel_size": 3} + + # 2. Without grid_keys: returns all primitive scalar attributes + tags_all = extract_config_tags(model) + assert tags_all == {"hidden_dim": 32, "kernel_size": 3, "n_layers": 2} + + # 3. None or non-config object + assert extract_config_tags(None) == {} + assert extract_config_tags("string") == {} + + +def test_get_experiment_configs_detection_logging(tmp_path, caplog): + """Verify that get_experiment_configs logs detected grid search configurations at INFO level.""" + run_dir = tmp_path / "test_run" + run_dir.mkdir() + jobs_file = run_dir / "jobs.jsonl" + records = [ + {"job_id": "j1", "tags": {"lr": 0.01, "hidden_dim": 32}}, + {"job_id": "j2", "tags": {"lr": 0.02, "hidden_dim": 64}}, + {"job_id": "j3", "tags": None}, + ] + with jobs_file.open("w") as f: + for r in records: + f.write(json.dumps(r) + "\n") + + with caplog.at_level(logging.INFO): + configs = get_experiment_configs(run_dir) + assert len(configs) == 2 + assert "Detected 2 grid search configuration(s)" in caplog.text + assert "hidden_dim, lr" in caplog.text + + +def test_build_dag_json_path_display(tmp_path): + """Verify build_dag_json populates path, display_path, and config tags.""" + model = DummyModel.C(hidden_dim=64, kernel_size=5, n_layers=1) + task = DummyTask() + + model_id = "model_" + "a" * 60 + task_id = "task_" + "b" * 60 + + full_jobs = {model_id: model, task_id: task} + dependencies = {task_id: [model_id]} + job_tags_dict = {task_id: {"lr": 0.01, "hidden_dim": 64}} + + workspace = tmp_path / "my_workspace" + run_dir = workspace / "experiments" / "test_exp" / "20261016_120000" + run_dir.mkdir(parents=True) + + jobs_file = run_dir / "jobs.jsonl" + with jobs_file.open("w") as f: + f.write( + json.dumps( + { + "job_id": task_id, + "task_id": "dummy.package.task", + "tags": {"lr": 0.01, "hidden_dim": 64}, + } + ) + + "\n" + ) + + dag_json = build_dag_json( + full_jobs, + dependencies, + job_tags_dict, + run_dir=run_dir, + workspace=str(workspace), + ) + + nodes_by_id = {n["id"]: n for n in dag_json["nodes"]} + task_node = nodes_by_id[task_id] + model_node = nodes_by_id[model_id] + + expected_full_path = str(workspace / "jobs" / "dummy.package.task" / task_id) + assert task_node["path"] == expected_full_path + assert task_node["locator"] == expected_full_path + assert task_node["display_path"] == f".../dummy.package.task/{task_id[:20]}..." + + assert model_node["tags"] == {"hidden_dim": 64} + assert model_node["path"] == "" + assert model_node["display_path"] == "" + + +def test_web_dag_renders_path_in_drawer(): + """Verify generated HTML drawer contains Path row and display_path.""" + dag_json = { + "experiment_id": "exp_test", + "run_id": "run_test", + "workspace": "/tmp/workspace", + "nodes": [ + { + "id": "job_12345678901234567890", + "name": "Learn", + "task_id": "my_package.learn", + "type": "task", + "status": "done", + "tags": {"lr": 0.001, "hidden_dim": 32}, + "path": "/tmp/workspace/jobs/my_package.learn/job_12345678901234567890", + "display_path": ".../my_package.learn/job_1234567890123456...", + "locator": "/tmp/workspace/jobs/my_package.learn/job_12345678901234567890", + "depends_on": [], + "depth": 0, + "x": 0.0, + "y": 0.0, + } + ], + "edges": [], + "configs": [{"lr": 0.001, "hidden_dim": 32}], + } + + html = generate_dag_html(dag_json) + assert 'Path' in html + assert "copyText" in html + assert "node.display_path" in html + + +def test_draw_dag_cli_save_as(tmp_path, monkeypatch): + """Verify draw_dag --save-as auto-detects extensions and saves without launching server.""" + full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()} + dependencies = {"j_train": ["j_root"]} + job_tags_dict = {"j_train": {"lr": 0.001}} + + run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000" + run_dir.mkdir(parents=True) + + monkeypatch.setattr( + draw_dag, + "resolve_run_dir", + lambda ws, exp_id, run_id: ("20260101_120000", run_dir), + ) + monkeypatch.setattr( + draw_dag, + "load_run_graph", + lambda r_dir: (full_jobs, dependencies, job_tags_dict), + ) + + server_called = [] + monkeypatch.setattr( + draw_dag, + "serve_dag_web", + lambda *args, **kwargs: server_called.append(True), + ) + + # 1. Test --save-as with .html: saves HTML and does NOT launch server even if --web was passed + html_out = tmp_path / "subfolder" / "dag.html" + monkeypatch.setattr( + "sys.argv", + ["draw_dag", "--web", "--save-as", str(html_out)], + ) + draw_dag.main() + assert html_out.exists() + assert "" in html_out.read_text() + assert len(server_called) == 0, "Server should not be launched when --save-as ends in .html" + + # 2. Test --save-as with .json + json_out = tmp_path / "subfolder" / "dag.json" + monkeypatch.setattr( + "sys.argv", + ["draw_dag", "--save-as", str(json_out)], + ) + draw_dag.main() + assert json_out.exists() + assert "j_train" in json_out.read_text() + + +def test_web_dag_template_loading(): + """Verify that the dedicated HTML template file loads correctly.""" + from experimaestro.tools.xpm_dag.xpm_web_dag import ( + HTML_TEMPLATE, + _load_html_template, + ) + + loaded = _load_html_template() + assert len(loaded) > 0 + assert loaded == HTML_TEMPLATE + assert "" in loaded + assert "__TITLE__" in loaded + assert "__RUN_ID__" in loaded + assert "__DEFAULT_LAYOUT__" in loaded + assert "__GRAPH_DATA__" in loaded + diff --git a/src/experimaestro/tools/xpm_dag/__init__.py b/src/experimaestro/tools/xpm_dag/__init__.py new file mode 100644 index 00000000..f0f4e226 --- /dev/null +++ b/src/experimaestro/tools/xpm_dag/__init__.py @@ -0,0 +1,57 @@ +"""Experimaestro DAG visualization, analysis, and export utilities.""" + +from .draw_dag import ( + main, + plot_dag, + print_robust_tree, +) +from .xpm_dag import ( + BOX_H, + BOX_W, + MARGIN_X, + MARGIN_Y, + build_dag_json, + compute_dag_depths, + compute_dag_layout, + extract_config_tags, + filter_subgraph_by_config, + filter_tasks_only, + get_all_dependencies, + get_experiment_configs, + load_dag_from_json, + load_run_graph, + load_run_status, + parse_filter_arg, + resolve_run_dir, + resolve_target_config, +) +from .xpm_web_dag import ( + generate_dag_html, + serve_dag_web, +) + +__all__ = [ + "BOX_H", + "BOX_W", + "MARGIN_X", + "MARGIN_Y", + "build_dag_json", + "compute_dag_depths", + "compute_dag_layout", + "extract_config_tags", + "filter_subgraph_by_config", + "filter_tasks_only", + "generate_dag_html", + "get_all_dependencies", + "get_experiment_configs", + "load_dag_from_json", + "load_run_graph", + "load_run_status", + "main", + "parse_filter_arg", + "plot_dag", + "print_robust_tree", + "resolve_run_dir", + "resolve_target_config", + "serve_dag_web", +] diff --git a/src/experimaestro/tools/xpm_dag/__main__.py b/src/experimaestro/tools/xpm_dag/__main__.py new file mode 100644 index 00000000..1e66431c --- /dev/null +++ b/src/experimaestro/tools/xpm_dag/__main__.py @@ -0,0 +1,6 @@ +"""Entrypoint for `python -m experimaestro.tools.xpm_dag`.""" + +from .draw_dag import main + +if __name__ == "__main__": + main() diff --git a/src/experimaestro/tools/xpm_dag/dag_template.html b/src/experimaestro/tools/xpm_dag/dag_template.html new file mode 100644 index 00000000..702e6472 --- /dev/null +++ b/src/experimaestro/tools/xpm_dag/dag_template.html @@ -0,0 +1,1501 @@ + + + + + + __TITLE__ + + + + + + + + +
+
+

__TITLE__

+ __RUN_ID__ +
+ +
+ +
+ +
+
+ + +
+ + + + + +
+
+ + +
+ + + + + + + + + + + + + + + + + +
+ + +
+
+
Task Inspector
+ +
+
+ +
+
+ +
Copied to clipboard
+ + + + diff --git a/src/experimaestro/tools/xpm_dag/draw_dag.py b/src/experimaestro/tools/xpm_dag/draw_dag.py new file mode 100644 index 00000000..95b9ab47 --- /dev/null +++ b/src/experimaestro/tools/xpm_dag/draw_dag.py @@ -0,0 +1,591 @@ +"""Draw the dependency DAG of an Experimaestro experiment run as a tree, plot, or web page. + +Run with:: + + # Interactive DAG web visualization in browser: + uv run python -m experimaestro.tools.xpm_dag --web + + # Export standalone interactive HTML report (auto-detected, zero-server): + uv run python -m experimaestro.tools.xpm_dag --save-as dag.html + + # Export standardized JSON graph: + uv run python -m experimaestro.tools.xpm_dag --save-as dag.json + + # Text tree terminal output: + uv run python -m experimaestro.tools.xpm_dag --no-plot + + # Interactive matplotlib plot: + uv run python -m experimaestro.tools.xpm_dag + + # Save plot directly to image without GUI window: + uv run python -m experimaestro.tools.xpm_dag --save-as dag.png + + # List all hyperparameter configurations: + uv run python -m experimaestro.tools.xpm_dag --list-configs + + # Inspect a single configuration: + uv run python -m experimaestro.tools.xpm_dag --single-config -c 0 + uv run python -m experimaestro.tools.xpm_dag --filter "lr=0.001,kernel_size=3" + +This module provides the main CLI entrypoint and coordinates terminal ASCII, +matplotlib, and interactive web representations of the experiment DAG. +""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path +import networkx as nx + +from experimaestro import tags + +# Re-export core DAG utilities and models for display styles & test compatibility +from .xpm_dag import ( + BOX_H, + BOX_W, + MARGIN_X, + MARGIN_Y, + build_dag_json, + compute_dag_depths, + compute_dag_layout, + extract_config_tags, + filter_subgraph_by_config, + filter_tasks_only, + get_all_dependencies, + get_experiment_configs, + load_dag_from_json, + load_run_graph, + load_run_status, + parse_filter_arg, + resolve_run_dir, + resolve_target_config, +) +from .xpm_web_dag import ( + generate_dag_html, + serve_dag_web, +) + +__all__ = [ + "BOX_H", + "BOX_W", + "MARGIN_X", + "MARGIN_Y", + "build_dag_json", + "compute_dag_depths", + "compute_dag_layout", + "extract_config_tags", + "filter_subgraph_by_config", + "filter_tasks_only", + "generate_dag_html", + "get_all_dependencies", + "get_experiment_configs", + "load_dag_from_json", + "load_run_graph", + "load_run_status", + "parse_filter_arg", + "plot_dag", + "print_robust_tree", + "resolve_run_dir", + "resolve_target_config", + "serve_dag_web", +] + + +def print_robust_tree( + job_ids: list[str], + dependents: dict[str, list[str]], + dependencies: dict[str, list[str]], + info_jobs: dict, + job_tags_dict: dict, + visited: set[str], + depths: dict[str, int], + pos: dict[str, tuple[float, float]], + prefix: str = "", + is_last: bool = True, +) -> None: + """Recursively print the dependency tree with individual nodes, depth and Y info.""" + if not job_ids: + return + + def job_sort_key(jid: str): + config = info_jobs.get(jid) + return ( + depths.get(jid, 0), + -pos.get(jid, (0, 0))[1], + type(config).__name__ if config else "", + jid, + ) + + sorted_job_ids = sorted(job_ids, key=job_sort_key) + + for idx, jid in enumerate(sorted_job_ids): + is_last_node = is_last and (idx == len(sorted_job_ids) - 1) + config = info_jobs.get(jid) + job_type = type(config).__name__.replace(".XPMConfig", "") if config else "Unknown" + depth = depths.get(jid, 0) + y_coord = pos.get(jid, (0, 0))[1] + + # Use tags from jobs.jsonl, fallback to tags(config) + jtags = job_tags_dict.get(jid) or (tags(config) if config else {}) + tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items()) + label = f"{job_type} [{jid[:8]}] (depth {depth}, y {y_coord:.1f})" + if tag_str: + label += f" ({tag_str})" + + already_visited = jid in visited + if already_visited: + label += " (already shown)" + + connector = "└── " if is_last_node else "├── " + print(f"{prefix}{connector}{label}") # noqa: T201 + + if already_visited: + continue + + visited.add(jid) + child_prefix = prefix + (" " if is_last_node else "│ ") + next_job_ids = sorted(dependents.get(jid, []), key=job_sort_key) + if next_job_ids: + print_robust_tree( + next_job_ids, + dependents, + dependencies, + info_jobs, + job_tags_dict, + visited, + depths, + pos, + child_prefix, + True, + ) + + +def plot_dag( + full_jobs: dict, + job_tags_dict: dict, + pos: dict[str, tuple[float, float]], + reduced_dag: nx.DiGraph, + box_w: float = BOX_W, + box_h: float = BOX_H, + title_suffix: str = "", + show_plot: bool = True, + save_path: Path | None = None, +) -> None: + """Plot the DAG hierarchically using pre-computed layout and reduced graph. + + Args: + full_jobs: Mapping of job ID to config/task instance. + job_tags_dict: Mapping of job ID to tags dictionary. + pos: Node coordinates mapping job ID to (x, y). + reduced_dag: Transitive reduction of the DAG. + box_w: Width of node boxes. + box_h: Height of node boxes. + title_suffix: Optional suffix appended to figure title (e.g. config details). + show_plot: Whether to display the plot interactively. + save_path: Optional path to save the figure image. + """ + import matplotlib.pyplot as plt + from matplotlib.patches import FancyBboxPatch + from matplotlib.lines import Line2D + from experimaestro import Task, Action, tags + + node_info = {} + for jid, config in full_jobs.items(): + name = type(config).__name__.replace(".XPMConfig", "") + ntype = ( + "action" + if isinstance(config, Action) + else "task" if isinstance(config, Task) else "config" + ) + + # Get tags and create label + jtags = job_tags_dict.get(jid) or (tags(config) if config else {}) + tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items()) + + node_info[jid] = { + "name": name, + "hash": f"[{jid[:8]}]", + "tags": tag_str if tag_str else None, + "type": ntype, + } + + # Calculate dynamic figure size based on the layout + all_x = [p[0] for p in pos.values()] + all_y = [p[1] for p in pos.values()] + min_x, max_x = min(all_x), max(all_x) + min_y, max_y = min(all_y), max(all_y) + + # Scale factor: 1 unit = 0.8 inches + scale = 0.8 + fig_w = max(10, (max_x - min_x + box_w * 2) * scale) + fig_h = max(6, (max_y - min_y + box_h * 2) * scale) + + plt.figure(figsize=(fig_w, fig_h)) + ax = plt.gca() + title = "Experimaestro DAG (Hierarchical Top-Aligned Flow)" + if title_suffix: + title += f" - {title_suffix}" + plt.title(title) + + for u, v in reduced_dag.edges(): + (x1, y1), (x2, y2) = pos[u], pos[v] + plt.annotate( + "", + xy=(x2 - box_w / 2, y2), + xytext=(x1 + box_w / 2, y1), + zorder=1, + arrowprops=dict( + arrowstyle="->", + color="black", + alpha=1.0, + shrinkA=0, + shrinkB=0, + linewidth=1.5, + ), + ) + + styles = {"config": "#e6ffed", "task": "#e6f4ff", "action": "#fff1f0"} + for jid, (x, y) in pos.items(): + info = node_info[jid] + ax.add_patch( + FancyBboxPatch( + (x - box_w / 2, y - box_h / 2), + box_w, + box_h, + boxstyle="round,pad=0.03,rounding_size=0.2", + facecolor=styles[info["type"]], + edgecolor="#666666", + linewidth=2.0, + zorder=2, + ) + ) + if info["tags"]: + plt.text( + x, + y + 0.25, + info["name"], + ha="center", + va="center", + fontsize=10.5, + fontweight="bold", + color="#111111", + zorder=3, + ) + plt.text( + x, + y + 0.03, + info["hash"], + ha="center", + va="center", + fontsize=7.5, + color="#777777", + zorder=3, + ) + plt.text( + x, + y - 0.23, + info["tags"], + ha="center", + va="center", + fontsize=8.5, + fontweight="bold", + color="#222222", + zorder=3, + ) + else: + plt.text( + x, + y + 0.14, + info["name"], + ha="center", + va="center", + fontsize=10.5, + fontweight="bold", + color="#111111", + zorder=3, + ) + plt.text( + x, + y - 0.14, + info["hash"], + ha="center", + va="center", + fontsize=7.5, + color="#777777", + zorder=3, + ) + + plt.legend( + handles=[ + Line2D( + [0], + [0], + marker="s", + color="w", + label=k.capitalize(), + markerfacecolor=v, + markersize=10, + markeredgecolor="gray", + ) + for k, v in styles.items() + ], + loc="upper left", + ) + + plt.xlim(min_x - box_w, max_x + box_w) + plt.ylim(min_y - box_h, max_y + box_h) + plt.axis("off") + plt.tight_layout() + if save_path: + plt.savefig(save_path, bbox_inches="tight") + print(f"Saved DAG plot to {save_path}") # noqa: T201 + if show_plot: + plt.show() + plt.close() + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--workspace", + type=str, + default=None, + help="Workspace ID or directory path (default: auto-detected via find_workspace).", + ) + parser.add_argument( + "--experiment-id", + default="MNIST_train_advanced", + help="Experiment identifier (default: %(default)s).", + ) + parser.add_argument( + "--run-id", + default=None, + help="Specific run timestamp ID (default: most recent run).", + ) + parser.add_argument( + "--tasks-only", + action="store_true", + help="Only display Task and Action nodes, skipping static config nodes.", + ) + parser.add_argument( + "--web", + action="store_true", + help="Launch interactive DAG web viewer and open in browser.", + ) + parser.add_argument( + "--html", + type=Path, + default=None, + help="Path where the standalone interactive DAG HTML page should be saved (e.g. dag.html).", + ) + parser.add_argument( + "--json", + nargs="?", + const="", + default=None, + help="Export DAG as JSON to stdout or file (compatible with Experimaestro web CLI).", + ) + parser.add_argument( + "--no-plot", + action="store_false", + dest="plot", + default=True, + help="Skip the matplotlib interactive plot and only output the text tree.", + ) + parser.add_argument( + "--list-configs", + action="store_true", + help="List all unique configuration parameter sets found in the experiment and exit.", + ) + parser.add_argument( + "--single-config", + action="store_true", + help="Render the DAG for a single grid configuration (defaults to config index 0).", + ) + parser.add_argument( + "-c", + "--config-index", + type=int, + default=None, + help="Index of the configuration to display (see --list-configs). Implies --single-config.", + ) + parser.add_argument( + "--filter", + default=None, + help="Filter the DAG by tag values (e.g. 'lr=0.001,kernel_size=3').", + ) + parser.add_argument( + "--include-actions", + action="store_true", + help="Include downstream Action and EvaluatedModel nodes when filtering by configuration.", + ) + parser.add_argument( + "--layout", + choices=["dagre", "aligned"], + default="dagre", + help="Layout algorithm for interactive web view: 'dagre' (default) or 'aligned' (stage-aligned flow).", + ) + parser.add_argument( + "--save-as", + type=Path, + default=None, + help="Path where the DAG output should be saved (auto-detects .html, .json, .png, etc.). " + "If ending in .html, generates standalone HTML without launching the server.", + ) + args = parser.parse_args() + + save_plot_path = None + if args.save_as: + ext = args.save_as.suffix.lower() + if ext in (".html", ".htm"): + args.html = args.save_as + args.web = False + elif ext == ".json": + args.json = str(args.save_as) + args.web = False + elif ext in (".png", ".pdf", ".svg", ".jpg", ".jpeg"): + save_plot_path = args.save_as + args.plot = False + args.web = False + else: + logging.error( + "Unsupported file extension '%s' for --save-as. " + "Supported extensions: .html, .htm, .json, .png, .pdf, .svg, .jpg", + ext, + ) + return + + run_info = resolve_run_dir(args.workspace, args.experiment_id, args.run_id) + if run_info is None: + return + run_id, run_dir = run_info + + if args.list_configs: + configs = get_experiment_configs(run_dir) + if not configs: + print(f"No configuration tags found for experiment '{args.experiment_id}' ({run_id}).") # noqa: T201 + return + print(f"Available configurations in experiment '{args.experiment_id}' (run {run_id}):") # noqa: T201 + for idx, cfg_dict in enumerate(configs): + cfg_str = ", ".join(f"{k}={v}" for k, v in sorted(cfg_dict.items())) + print(f" [{idx}] {cfg_str}") # noqa: T201 + return + + graph_data = load_run_graph(run_dir) + if graph_data is None: + return + full_jobs, dependencies, job_tags_dict = graph_data + + title_suffix = "" + if args.single_config or args.config_index is not None or args.filter: + configs = get_experiment_configs(run_dir) + target_tags, title_suffix = resolve_target_config(args, configs) + if target_tags is None: + return + full_jobs, dependencies = filter_subgraph_by_config( + full_jobs, + dependencies, + job_tags_dict, + target_tags, + include_actions=args.include_actions, + ) + + if args.tasks_only: + full_jobs, dependencies = filter_tasks_only(full_jobs, dependencies) + + workspace_path = str(run_dir.parents[2]) if len(run_dir.parents) >= 3 else "" + + # Export JSON if requested + if args.json is not None: + dag_json = build_dag_json( + full_jobs, + dependencies, + job_tags_dict, + run_dir=run_dir, + experiment_id=args.experiment_id, + run_id=run_id, + workspace=workspace_path, + ) + json_str = json.dumps(dag_json, indent=2) + if args.json: # Target file provided + out_path = Path(args.json) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json_str) + print(f"Saved DAG JSON to {out_path}") # noqa: T201 + else: + print(json_str) # noqa: T201 + return + + # Export standalone HTML page if requested + if args.html is not None: + dag_json = build_dag_json( + full_jobs, + dependencies, + job_tags_dict, + run_dir=run_dir, + experiment_id=args.experiment_id, + run_id=run_id, + workspace=workspace_path, + ) + html_str = generate_dag_html( + dag_json, + title=f"Experimaestro DAG - {args.experiment_id}", + default_layout=args.layout, + ) + args.html.parent.mkdir(parents=True, exist_ok=True) + args.html.write_text(html_str) + print(f"Saved interactive DAG HTML to {args.html}") # noqa: T201 + return + + # Launch interactive web server if requested + if args.web: + dag_json = build_dag_json( + full_jobs, + dependencies, + job_tags_dict, + run_dir=run_dir, + experiment_id=args.experiment_id, + run_id=run_id, + workspace=workspace_path, + ) + serve_dag_web(dag_json, default_layout=args.layout) + return + + # Default / Matplotlib & Text Tree visualization + dependents: dict[str, list[str]] = {} + for jid, deps in dependencies.items(): + for d in deps: + dependents.setdefault(d, []).append(jid) + + depths, pos, reduced_dag = compute_dag_layout(full_jobs, dependencies, dependents) + + tree_roots = [jid for jid in full_jobs if not dependencies.get(jid)] + print_robust_tree( + tree_roots, + dependents, + dependencies, + full_jobs, + job_tags_dict, + set(), + depths, + pos, + ) + + if args.plot or save_plot_path: + plot_dag( + full_jobs, + job_tags_dict, + pos, + reduced_dag, + title_suffix=title_suffix, + show_plot=args.plot, + save_path=save_plot_path, + ) + + +if __name__ == "__main__": + main() diff --git a/src/experimaestro/tools/xpm_dag/xpm_dag.py b/src/experimaestro/tools/xpm_dag/xpm_dag.py new file mode 100644 index 00000000..03cd759a --- /dev/null +++ b/src/experimaestro/tools/xpm_dag/xpm_dag.py @@ -0,0 +1,663 @@ +"""Core DAG utilities, graph algorithms, and standardized JSON serialization. + +This module provides display-agnostic utilities for reading Experimaestro +experiment graphs from workspace records (``objects.jsonl``, ``jobs.jsonl``, +and ``status.json``), computing hierarchical forward topological layouts, +filtering subgraphs by hyperparameter configurations, and serializing the DAG +into a standardized dual-compatible JSON format. +""" + +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path +import re +from typing import Any +import networkx as nx + +from experimaestro import tags +from experimaestro.core.objects.config import ConfigInformation +from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider +from experimaestro.settings import find_workspace, get_workspace + +# Default layout sizing constants +BOX_W = 25.0 +BOX_H = 1.05 +MARGIN_X = 4.0 # Gap between boxes horizontally +MARGIN_Y = 0.55 # Gap between boxes vertically + + +def resolve_run_dir( + workspace_arg: str | Path | None, experiment_id: str, run_id: str | None +) -> tuple[str, Path] | None: + """Find and return (run_id, run_dir) from the workspace. + + Uses the explicitly specified workspace if provided (by ID or path). + Otherwise, auto-selects the default workspace just as in real experiments + using Experimaestro's workspace triggers matching the experiment ID. + """ + if workspace_arg: + ws_id = str(workspace_arg) + if get_workspace(ws_id) is not None: + ws_env = find_workspace(workspace=ws_id, experiment_id=experiment_id) + else: + ws_path = Path(workspace_arg).expanduser() + ws_env = find_workspace(workdir=ws_path, experiment_id=experiment_id) + else: + ws_env = find_workspace(experiment_id=experiment_id) + + workspace = ws_env.path + experiments_dir = workspace / "experiments" + if not experiments_dir.exists(): + logging.error("No 'experiments' directory found in workspace '%s'.", workspace) + return None + + available_xps = sorted( + [p.name for p in experiments_dir.iterdir() if p.is_dir() and not p.name.startswith(".")] + ) + exp_dir = experiments_dir / experiment_id + if not exp_dir.exists(): + xps_str = ", ".join(f"'{x}'" for x in available_xps) if available_xps else "(none)" + logging.error( + "Experiment '%s' not found in workspace '%s'. Available experiments: %s", + experiment_id, + workspace, + xps_str, + ) + return None + + available_runs = sorted( + [p.name for p in exp_dir.iterdir() if p.is_dir() and not p.name.startswith(".")] + ) + if not available_runs: + logging.error( + "Experiment '%s' in workspace '%s' has no recorded runs.", + experiment_id, + workspace, + ) + return None + + provider = WorkspaceStateProvider(workspace) + if run_id: + if run_id not in available_runs: + runs_str = ", ".join(f"'{r}'" for r in available_runs) + logging.error( + "Run '%s' not found for experiment '%s' in workspace '%s'. Available runs: %s", + run_id, + experiment_id, + workspace, + runs_str, + ) + return None + resolved_run_id = run_id + else: + try: + current_run = provider.get_current_run(experiment_id) + except Exception: + current_run = None + resolved_run_id = current_run or available_runs[-1] + + run_dir = exp_dir / resolved_run_id + if not run_dir.exists(): + logging.error("Run directory does not exist: %s", run_dir) + return None + + return resolved_run_id, run_dir + + +def extract_config_tags( + config: Any, grid_keys: set[str] | None = None +) -> dict[str, Any]: + """Extract hyperparameter tags from a Config node's xpmvalues. + + If grid_keys is provided, extracts parameters matching grid_keys. + Otherwise, extracts all scalar primitive parameters (int, float, str, bool). + """ + if config is None: + return {} + xpm = getattr(config, "__xpm__", None) + if xpm is None or not hasattr(xpm, "xpmvalues"): + return {} + + raw_params: dict[str, Any] = {} + try: + for param, val in xpm.xpmvalues(): + p_name = getattr(param, "name", str(param)) + if isinstance(val, (int, float, str, bool)) and not p_name.startswith("_"): + raw_params[p_name] = val + except Exception: + return {} + + if grid_keys: + return {k: v for k, v in raw_params.items() if k in grid_keys} + return raw_params + + +def load_run_graph( + run_dir: Path, +) -> tuple[dict, dict[str, list[str]], dict[str, dict]] | None: + """Load objects.jsonl and jobs.jsonl into full_jobs, dependencies, and job_tags_dict.""" + objects_file = run_dir / "objects.jsonl" + if not objects_file.exists(): + logging.error("Objects record not found: %s", objects_file) + return None + + with objects_file.open() as f: + all_definitions = [json.loads(line) for line in f if line.strip()] + all_configs_dict = ConfigInformation.load_objects( + all_definitions, as_instance=False + ) + + full_jobs = {} + for defn in all_definitions: + if "identifier" in defn: + jid = defn["identifier"] + config = all_configs_dict[defn["id"]] + full_jobs[jid] = config + if "init-tasks" in defn: + config.__xpm__.init_tasks = [ + all_configs_dict[tid] for tid in defn["init-tasks"] + ] + + dependencies = { + jid: get_all_dependencies(config, full_jobs) + for jid, config in full_jobs.items() + } + dependencies = {k: v for k, v in dependencies.items() if v} + + job_tags_dict = {} + grid_keys: set[str] = set() + jobs_jsonl = run_dir / "jobs.jsonl" + if jobs_jsonl.exists(): + with jobs_jsonl.open() as f: + for line in f: + if line.strip(): + job_data = json.loads(line) + if "job_id" in job_data and "tags" in job_data: + jtags = job_data["tags"] or {} + job_tags_dict[job_data["job_id"]] = jtags + if jtags: + grid_keys.update(jtags.keys()) + + # Populate tags for Config nodes (e.g. CNN architecture params) + for jid, config in full_jobs.items(): + if jid not in job_tags_dict and config: + cfg_tags = extract_config_tags(config, grid_keys if grid_keys else None) + if cfg_tags: + job_tags_dict[jid] = cfg_tags + + return full_jobs, dependencies, job_tags_dict + + +def load_run_status(run_dir: Path) -> dict[str, Any]: + """Load status.json if present in the run directory.""" + status_file = run_dir / "status.json" + if not status_file.exists(): + return {} + try: + with status_file.open() as f: + return json.load(f) + except Exception as e: + logging.warning("Could not read status.json at %s: %s", status_file, e) + return {} + + +def get_all_dependencies(config, full_jobs: dict) -> list[str]: + """Walk the config and its fields to find all dependencies.""" + xpm = config.__xpm__ + dep_ids = set() + current_id = str(xpm.identifier) + + if hasattr(xpm, "task") and xpm.task: + dep_ids.add(str(xpm.task.__xpm__.identifier)) + if hasattr(xpm, "init_tasks"): + for init_task in xpm.init_tasks: + if hasattr(init_task, "__xpm__"): + dep_ids.add(str(init_task.__xpm__.identifier)) + + for dep in getattr(xpm, "dependencies", []): + if hasattr(dep, "__identifier__"): + dep_ids.add(str(dep.__identifier__())) + elif hasattr(dep, "__xpm__"): + dep_ids.add(str(dep.__xpm__.identifier)) + else: + dep_ids.add(str(dep)) + + processed_configs = set() + + def walk_value(v): + from experimaestro import Config + + if isinstance(v, Config): + v_id = str(v.__xpm__.identifier) + if v_id in full_jobs: + dep_ids.add(v_id) + if v_id not in processed_configs: + processed_configs.add(v_id) + for _, sub_value in v.__xpm__.xpmvalues(): + walk_value(sub_value) + elif isinstance(v, list): + for item in v: + walk_value(item) + elif isinstance(v, dict): + for item in v.values(): + walk_value(item) + elif isinstance(v, (str, Path)) and "/jobs/" in str(v): + match = re.search(r"/jobs/[^/]+/([0-9a-f]{64})", str(v)) + if match: + v_id = match.group(1) + if v_id in full_jobs: + dep_ids.add(v_id) + + for name, value in xpm.xpmvalues(): + walk_value(value) + if current_id in dep_ids: + dep_ids.remove(current_id) + return list(dep_ids) + + +def get_experiment_configs(run_dir: Path) -> list[dict]: + """Retrieve unique non-empty tag dictionaries from jobs.jsonl in a run directory.""" + jobs_jsonl = run_dir / "jobs.jsonl" + if not jobs_jsonl.exists(): + return [] + + configs: list[dict] = [] + seen: set[tuple] = set() + with jobs_jsonl.open() as f: + for line in f: + if not line.strip(): + continue + d = json.loads(line) + tags_dict = d.get("tags") + if tags_dict: + key = tuple(sorted(tags_dict.items())) + if key not in seen: + seen.add(key) + configs.append(tags_dict) + if configs: + param_keys = sorted(set().union(*(c.keys() for c in configs))) + logging.info( + "Detected %d grid search configuration(s) (parameters: %s)", + len(configs), + ", ".join(param_keys), + ) + return configs + + +def parse_filter_arg(filter_str: str) -> dict[str, str | int | float]: + """Parse a comma-separated key=val filter string into typed values.""" + parsed: dict[str, str | int | float] = {} + for item in filter_str.split(","): + item = item.strip() + if not item or "=" not in item: + continue + k, v = item.split("=", 1) + k, v = k.strip(), v.strip() + try: + parsed[k] = int(v) + except ValueError: + try: + parsed[k] = float(v) + except ValueError: + parsed[k] = v + return parsed + + +def filter_subgraph_by_config( + full_jobs: dict, + dependencies: dict[str, list[str]], + job_tags_dict: dict[str, dict], + target_tags: dict, + include_actions: bool = False, +) -> tuple[dict, dict[str, list[str]]]: + """Extract a subgraph for a single configuration or a subset of configurations.""" + # 1. Identify jobs whose tags match all key-values in target_tags + matching = [ + jid + for jid, tags_val in job_tags_dict.items() + if jid in full_jobs + and all(tags_val.get(k) == v for k, v in target_tags.items()) + ] + if not matching: + return {}, {} + + # 2. Traverse upstream ancestors (CNN, Datasets, ParameterLoader) + selected_nodes = set(matching) + stack = list(matching) + while stack: + curr = stack.pop() + for parent in dependencies.get(curr, []): + if parent in full_jobs and parent not in selected_nodes: + selected_nodes.add(parent) + stack.append(parent) + + # 3. If requested, include downstream actions and evaluated models that depend on matching jobs + if include_actions: + from experimaestro import Action + + for jid, config in full_jobs.items(): + if isinstance(config, Action) or type(config).__name__.startswith("EvaluatedModel"): + deps = dependencies.get(jid, []) + if any(m in deps for m in matching): + selected_nodes.add(jid) + + # 4. Construct filtered jobs and dependencies restricted to selected_nodes + filtered_jobs = {jid: full_jobs[jid] for jid in selected_nodes} + filtered_dependencies = { + jid: [p for p in dependencies.get(jid, []) if p in selected_nodes] + for jid in selected_nodes + } + filtered_dependencies = {k: v for k, v in filtered_dependencies.items() if v} + return filtered_jobs, filtered_dependencies + + +def filter_tasks_only( + full_jobs: dict, dependencies: dict[str, list[str]] +) -> tuple[dict, dict[str, list[str]]]: + """Filter the graph to include only Task and Action nodes.""" + from experimaestro import Task, Action + + is_task = {jid: isinstance(c, (Task, Action)) for jid, c in full_jobs.items()} + task_deps: dict[str, list[str]] = {} + for jid in full_jobs: + if not is_task[jid]: + continue + tdeps, q, processed = set(), list(dependencies.get(jid, [])), set() + while q: + d = q.pop() + if d in processed: + continue + processed.add(d) + if is_task.get(d): + tdeps.add(d) + else: + q.extend(dependencies.get(d, [])) + task_deps[jid] = list(tdeps) + filtered_jobs = {jid: c for jid, c in full_jobs.items() if is_task[jid]} + return filtered_jobs, task_deps + + +def compute_dag_depths( + dag: nx.DiGraph, + full_jobs: dict, + align_types: bool = True, + aligned_class_names: set[str] | None = None, +) -> dict[str, int]: + """Compute horizontal depths for nodes in a DAG with robust stage alignment. + + Implements a 3-pass algorithm: + 1. Base Forward Topological Leveling: + depth(v) = max_{u in deps(v)}(depth(u) + 1), roots at 0. + 2. Grid Stage Alignment: + Aligns tasks of each type in aligned_class_names to max depth across peers. + 3. Deep Dependency Forward Push: + Topological push sweep ensuring depth(succ) >= depth(node) + 1 for all edges. + """ + if aligned_class_names is None: + aligned_class_names = {"Learn", "ParameterLoader", "Evaluate"} + + # Pass 1: Base Forward Topological Leveling + depths: dict[str, int] = {} + topo_order = list(nx.topological_sort(dag)) + for node in topo_order: + preds = list(dag.predecessors(node)) + if not preds: + depths[node] = 0 + else: + depths[node] = max(depths[p] for p in preds) + 1 + + # Pass 2: Grid Stage Alignment + if align_types: + for tname in aligned_class_names: + nodes_t = [ + n + for n in dag.nodes + if n in full_jobs + and type(full_jobs[n]).__name__.split(".")[0] == tname + ] + if nodes_t: + max_d = max(depths[n] for n in nodes_t) + for n in nodes_t: + depths[n] = max_d + + # Pass 3: Deep Dependency Forward Push + for node in topo_order: + for succ in dag.successors(node): + if depths[succ] <= depths[node]: + depths[succ] = depths[node] + 1 + + return depths + + +def compute_dag_layout( + full_jobs: dict, + dependencies: dict[str, list[str]], + dependents: dict[str, list[str]], + align_types: bool = True, +) -> tuple[dict[str, int], dict[str, tuple[float, float]], nx.DiGraph]: + """Compute depths and (X, Y) coordinates for the DAG to maximize straight arrows.""" + dag = nx.DiGraph() + for jid in full_jobs: + dag.add_node(jid) + for u, targets in dependents.items(): + for v in targets: + if u in dag and v in dag: + dag.add_edge(u, v) + + depths = compute_dag_depths(dag, full_jobs, align_types=align_types) + + reduced_dag = nx.transitive_reduction(dag) + preds: dict[str, list[str]] = {n: [] for n in full_jobs} + for u, v in reduced_dag.edges(): + preds[v].append(u) + + pos: dict[str, tuple[float, float]] = {} + unique_depths = sorted(set(depths.values())) + nodes_per_type: dict[str, list[str]] = {} + for d in unique_depths: + nodes_at_depth = [jid for jid, depth in depths.items() if depth == d] + + # Compute desired Y (average of parents) + desired_y: dict[str, float] = {} + for jid in nodes_at_depth: + p_ys = [pos[p][1] for p in preds[jid] if p in pos] + tname = type(full_jobs[jid]).__name__.split(".")[0] + sibling_ys = [pos[s][1] for s in nodes_per_type.get(tname, []) if s in pos] + p_ys.extend(sibling_ys) + desired_y[jid] = sum(p_ys) / len(p_ys) if p_ys else 0.0 + + nodes_at_depth.sort( + key=lambda j: (-desired_y[j], type(full_jobs[j]).__name__, j) + ) + + for i, jid in enumerate(nodes_at_depth): + pos[jid] = (d * (BOX_W + MARGIN_X), -i * (BOX_H + MARGIN_Y)) + + for jid in nodes_at_depth: + tname = type(full_jobs[jid]).__name__.split(".")[0] + nodes_per_type.setdefault(tname, []).append(jid) + + return depths, pos, reduced_dag + + +def resolve_target_config( + args: argparse.Namespace, + configs: list[dict], +) -> tuple[dict | None, str]: + """Resolve target configuration tags and title suffix from CLI options.""" + if args.config_index is not None: + if not (0 <= args.config_index < len(configs)): + logging.error( + "Config index %d out of range [0..%d]. Use --list-configs to see options.", + args.config_index, + len(configs) - 1, + ) + return None, "" + target_tags = configs[args.config_index] + return target_tags, ", ".join(f"{k}={v}" for k, v in sorted(target_tags.items())) + + if args.filter: + filter_tags = parse_filter_arg(args.filter) + if args.single_config: + for cfg_cand in configs: + if all(cfg_cand.get(k) == v for k, v in filter_tags.items()): + return cfg_cand, ", ".join(f"{k}={v}" for k, v in sorted(cfg_cand.items())) + logging.error("No configuration matched filter: %s", args.filter) + return None, "" + return filter_tags, f"Filter: {args.filter}" + + if args.single_config: + if not configs: + logging.error("No configurations found to filter.") + return None, "" + return configs[0], ", ".join(f"{k}={v}" for k, v in sorted(configs[0].items())) + + return None, "" + + +def build_dag_json( + full_jobs: dict, + dependencies: dict[str, list[str]], + job_tags_dict: dict[str, dict], + run_dir: Path | None = None, + experiment_id: str = "", + run_id: str = "", + workspace: str = "", +) -> dict[str, Any]: + """Serialize the experiment DAG into a standardized, dual-compatible JSON format. + + Supports both snake_case and camelCase field aliases so it can be consumed + transparently by draw_dag web renderers, experimaestro CLI, and web UI. + """ + from experimaestro import Task, Action + + # Read jobs metadata from jobs.jsonl and status.json if available + job_task_ids: dict[str, str] = {} + job_timestamps: dict[str, float] = {} + if run_dir: + jobs_jsonl = run_dir / "jobs.jsonl" + if jobs_jsonl.exists(): + with jobs_jsonl.open() as f: + for line in f: + if line.strip(): + try: + jd = json.loads(line) + jid = jd.get("job_id") + if jid: + if "task_id" in jd: + job_task_ids[jid] = jd["task_id"] + if "timestamp" in jd: + job_timestamps[jid] = jd["timestamp"] + except json.JSONDecodeError: + pass + + workspace_val = str(workspace) if workspace else "" + if not workspace_val and run_dir and len(run_dir.parents) >= 3: + workspace_val = str(run_dir.parents[2]) + + status_data = load_run_status(run_dir) if run_dir else {} + job_states: dict[str, str] = status_data.get("job_states", {}) + + configs = get_experiment_configs(run_dir) if run_dir else [] + grid_keys = set().union(*(c.keys() for c in configs)) if configs else None + + # Compute deterministic layout matching flat plot + dependents: dict[str, list[str]] = {} + for jid, deps in dependencies.items(): + for d in deps: + dependents.setdefault(d, []).append(jid) + + depths, pos, reduced_dag = compute_dag_layout(full_jobs, dependencies, dependents) + + nodes: list[dict[str, Any]] = [] + for jid, config in full_jobs.items(): + name = type(config).__name__.replace(".XPMConfig", "") + is_action = isinstance(config, Action) + is_task = isinstance(config, Task) + ntype = "action" if is_action else "task" if is_task else "config" + + # Determine task status + if jid in job_states: + status = job_states[jid] + elif is_task or is_action: + status = "done" # If in finished run but not explicitly state-tracked + else: + status = "config" + + # Tags + jtags = job_tags_dict.get(jid) + if not jtags and config: + try: + jtags = dict(tags(config)) + except Exception: + jtags = {} + if not jtags and config: + jtags = extract_config_tags(config, grid_keys) + deps = dependencies.get(jid, []) + + task_id = job_task_ids.get(jid, name) + if is_task or is_action: + if workspace_val: + full_path = str(Path(workspace_val) / "jobs" / task_id / jid) + else: + full_path = f"jobs/{task_id}/{jid}" + display_path = f".../{task_id}/{jid[:20]}..." + else: + full_path = "" + display_path = "" + + coord = pos.get(jid, (0.0, 0.0)) + depth_val = depths.get(jid, 0) + row_val = round(-coord[1] / (BOX_H + MARGIN_Y)) if (BOX_H + MARGIN_Y) > 0 else 0 + + node_dict = { + # Dual identifier keys + "id": jid, + "jobId": jid, + # Dual task naming keys + "name": name, + "task_id": task_id, + "taskId": task_id, + "type": ntype, + "status": status, + "tags": jtags, + # Dual dependency keys + "depends_on": deps, + "dependsOn": deps, + "path": full_path, + "display_path": display_path, + "locator": full_path, + "timestamp": job_timestamps.get(jid), + # Layout coordinates matching flat plot + "depth": depth_val, + "col": depth_val, + "row": row_val, + "x": coord[0], + "y": coord[1], + } + nodes.append(node_dict) + + edges: list[dict[str, str]] = [ + {"source": u, "target": v} for u, v in reduced_dag.edges() + ] + + return { + "experiment_id": experiment_id, + "run_id": run_id, + "workspace": workspace_val, + "nodes": nodes, + "edges": edges, + "configs": configs, + } + + +def load_dag_from_json(path_or_str: str | Path) -> dict[str, Any]: + """Load an ExperimentGraph JSON from a file path or JSON string.""" + p = Path(path_or_str) if isinstance(path_or_str, (str, Path)) else None + if p and p.exists(): + with p.open() as f: + return json.load(f) + return json.loads(str(path_or_str)) diff --git a/src/experimaestro/tools/xpm_dag/xpm_web_dag.py b/src/experimaestro/tools/xpm_dag/xpm_web_dag.py new file mode 100644 index 00000000..225a991a --- /dev/null +++ b/src/experimaestro/tools/xpm_dag/xpm_web_dag.py @@ -0,0 +1,96 @@ +"""Interactive web DAG visualization generator and server for Experimaestro. + +This module generates a standalone, responsive, interactive HTML5 web page +displaying the experiment DAG with client-side Dagre layout, D3 zoom/pan, +custom card nodes, status coloring, Left-to-Right / Top-to-Bottom flow toggles, +real-time single-config trial filtering, lineage path highlighting, and an +integrated side inspector drawer. +""" + +from __future__ import annotations + +import http.server +from importlib.resources import files +import json +from pathlib import Path +import socketserver +import threading +from typing import Any +import webbrowser + + +def _load_html_template() -> str: + """Load the standalone interactive HTML template.""" + try: + return ( + files("experimaestro.tools.xpm_dag") + .joinpath("dag_template.html") + .read_text(encoding="utf-8") + ) + except (TypeError, ModuleNotFoundError, FileNotFoundError): + return (Path(__file__).parent / "dag_template.html").read_text(encoding="utf-8") + + +HTML_TEMPLATE = _load_html_template() + + +def generate_dag_html( + dag_json: dict[str, Any], + title: str = "", + default_layout: str = "dagre", +) -> str: + """Generate a self-contained interactive HTML web page for the experiment DAG.""" + exp_id = dag_json.get("experiment_id") or "Experiment" + run_id = dag_json.get("run_id") or "" + page_title = title or f"Experimaestro DAG - {exp_id}" + + html = HTML_TEMPLATE.replace("__TITLE__", page_title) + html = html.replace("__RUN_ID__", run_id if run_id else "latest") + html = html.replace("__DEFAULT_LAYOUT__", default_layout) + html = html.replace("__GRAPH_DATA__", json.dumps(dag_json)) + return html + + +def serve_dag_web( + dag_json: dict[str, Any], + port: int = 0, + open_browser: bool = True, + default_layout: str = "dagre", +) -> None: + """Start an ephemeral HTTP server to view the interactive DAG and open the browser.""" + html_content = generate_dag_html(dag_json, default_layout=default_layout).encode( + "utf-8" + ) + + class DAGRequestHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + self.send_response(200) + self.send_header("Content-type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(html_content))) + self.end_headers() + self.wfile.write(html_content) + + def log_message(self, format, *args): # noqa: A002 + # Suppress normal HTTP request logging + pass + + # Use ThreadingTCPServer with port 0 for automatic port selection + class ThreadedServer(socketserver.ThreadingTCPServer): + allow_reuse_address = True + + server = ThreadedServer(("127.0.0.1", port), DAGRequestHandler) + actual_port = server.server_address[1] + url = f"http://localhost:{actual_port}" + + print(f"Starting interactive DAG web server on {url}") # noqa: T201 + print("Press Ctrl+C to stop the server.") # noqa: T201 + + if open_browser: + threading.Thread(target=lambda: webbrowser.open(url), daemon=True).start() + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nShutting down web server...") # noqa: T201 + finally: + server.server_close() From 3c385b04919abf584f8855f88bb6e1fd942e571e Mon Sep 17 00:00:00 2001 From: Victor Morand Date: Mon, 19 Oct 2026 12:18:12 +0200 Subject: [PATCH 2/7] feat: new dag cli command with docs --- docs/Makefile | 2 +- docs/source/cli.md | 66 +++ docs/source/demo | 2 +- docs/source/img/DAG_example.png | Bin 0 -> 71050 bytes docs/source/interfaces.md | 17 + src/experimaestro/cli/__init__.py | 8 + src/experimaestro/cli/dag.py | 190 ++++++++ src/experimaestro/tests/test_xpm_dag.py | 199 ++++++++- src/experimaestro/tools/xpm_dag/__init__.py | 2 + src/experimaestro/tools/xpm_dag/draw_dag.py | 462 +++++++++++++------- src/experimaestro/tools/xpm_dag/xpm_dag.py | 49 ++- 11 files changed, 824 insertions(+), 173 deletions(-) create mode 100644 docs/source/img/DAG_example.png create mode 100644 src/experimaestro/cli/dag.py diff --git a/docs/Makefile b/docs/Makefile index 7fba6525..19e7509a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -4,7 +4,7 @@ # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= -SPHINXBUILD ?= uv run --project .. sphinx-build +SPHINXBUILD ?= uv run --project .. --extra docs sphinx-build SOURCEDIR = source BUILDDIR = build diff --git a/docs/source/cli.md b/docs/source/cli.md index cdd07bde..c7111075 100644 --- a/docs/source/cli.md +++ b/docs/source/cli.md @@ -474,3 +474,69 @@ the skill to the latest version. ## (Sphinx) Checking undocumented configurations Use `experimaestro check-documentation objects.inv my_package` to check for undocumented members. The `objects.inv` file should be generated by sphinx + +## Experiment DAG Visualization (`experimaestro dag`) + +Experimaestro provides dependency graph visualization and export capabilities via the `dag` command. It supports interactive browser rendering, standalone HTML reports, standardized JSON graphs, visual plots, and console text trees. + +![Experimaestro DAG Visualization](img/DAG_example.png) + +Principal Features: +- Two views available - a hiearchical-aligned heuristic (Aligned) and a more advanced using [Dagre JS](https://github.com/dagrejs/dagre) +- Optionally display intermediate configs (here `ParameterLoader` is just the output of `Learner` passed to `Evaluate`, not a task) or only the Tasks that are run. +- Shows the status of Tasks +- Wired with the [Grid Search](experiments/grid_search.md) feature to only show by default the first config - you can still choose to display all of them. + +Example usage: +```bash +# Launch interactive web viewer in browser (default mode) +experimaestro dag my_experiment + +# Export standalone interactive HTML report (zero-server requirement) +experimaestro dag my_experiment --save-as dag.html + +# Print terminal ASCII tree +experimaestro dag my_experiment --tree + +# Display interactive matplotlib plot window (requires matplotlib) +experimaestro dag my_experiment --plot + +# Save plot image directly without GUI window +experimaestro dag my_experiment --save-as dag.png + +# List hyperparameter trial configurations +experimaestro dag my_experiment --list-configs + +# Inspect a single trial configuration +experimaestro dag my_experiment -c 0 +experimaestro dag my_experiment --filter "lr=0.001,kernel_size=3" + +# Export standardized JSON graph +experimaestro dag my_experiment --save-as dag.json +experimaestro dag my_experiment --json +``` + +### Options + +| Option | Description | +|--------|-------------| +| `EXPERIMENT_ID` | Experiment identifier (positional or via `-e`, `--experiment`, `--experiment-id`). Auto-selected if only one experiment exists in the workspace | +| `-r, --run, --run-id` | Run timestamp ID (default: most recent run) | +| `-w, --workspace` | Workspace ID or path (default: auto-detected) | +| `-W, --web / -nw, --no-web` | Interactive web viewer in browser (default: enabled unless `--tree`, `--plot`, or file export is specified) | +| `-t, --tree` | Print text tree representation of the DAG to terminal | +| `--plot` | Display matplotlib interactive plot window (requires matplotlib) | +| `--port` | Port for web server (default: random available port) | +| `--no-open` | Do not automatically open browser when launching web viewer | +| `--save-as` | Save output file (`.html`, `.json`, `.png`, `.svg`, `.pdf`) | +| `--html` | Save standalone interactive HTML page | +| `--json` | Export DAG as JSON to standard output | +| `--json-file` | Export DAG as JSON to specified file | +| `--tasks-only` | Only display Task and Action nodes, skipping static config nodes | +| `--list-configs` | List all unique configuration parameter sets found in the experiment | +| `--single-config` | Render DAG for a single grid configuration (defaults to trial 0) | +| `-c, --config-index` | Index of configuration to display | +| `--filter` | Filter DAG by tag values (e.g. `lr=0.001,kernel_size=3`) | +| `--include-actions` | Include downstream Action and EvaluatedModel nodes when filtering configs | +| `--layout` | Web viewer layout algorithm: `dagre` (default) or `aligned` | + diff --git a/docs/source/demo b/docs/source/demo index cff06525..25283de3 160000 --- a/docs/source/demo +++ b/docs/source/demo @@ -1 +1 @@ -Subproject commit cff06525f7fd39ce24014f931759cfd99aa32b77 +Subproject commit 25283de340edb9254351829372708bef3a1ee059 diff --git a/docs/source/img/DAG_example.png b/docs/source/img/DAG_example.png new file mode 100644 index 0000000000000000000000000000000000000000..fab9084c91518aa5ec0805218cef6027abc61ae4 GIT binary patch literal 71050 zcmZ_01y~$Ew>XSLaca0zXp1aPakt{`Ebgw0m!d6J+@ZL;d-3AV;)^dXi_5~t?|tw6 zU-_OpPx2&_V>!v>WF|S8AIgf-IG7}u2nYx`vN95?2nZ-J1O!Ag478V$(z|z~FA1Wn zsbB)JO%<{m^A{*KPWGeuzZ2LO0_da`(OusFI{0@!$Yc>%2K0Cslf z7Yt@sF9$bc5VM0T?Z27)Cm#uOS5p^jCpT+H2daPg8k;z}y9v_J{3Gc9UH`79Imr6I zBssYLhgvTR0{(#j*jQKr|Bv?zslY#_e9G1!b2}XgYx|exd66N+!No1`FZ}->z4?+RJKdS$?rucU||Eu(+ zp@lF70ROMkgfQ#hf2%=25J8ZY5LE{u9%Z6uyp?>@fB2V|mN)2Y^lKcHS3gZe2;Q+# zi{dMzyt`F@D?xxPi!bpVh3ywz5C#Tjoa;C*)Gfct5YRDte7tr{;aTm|-U?Y$+}q!O znmCf5NH0C9wzUzXMSM;5e*_F0q*vct-I&N;{X+OZ!Z&IW>KF#3{{xEfDi{F~(~H{a zm$t08oHoQW>~%hzmaX zJw`-ZJB4mi36}qZ&M$4dTT3J<{2!^`Q$$kMRzRLuo5+2+vHw#~2_>q>>36&?n;CR6 z`1G=Iba4`|h<)D*HY_J^!V=3~Q&fIGoU@JlWPm9f_hsXl5&K3SiB1HGTH>3SAqF|q z`^5jL)*t>rzc3{^lXpRJrQzsUk^!io(p75@mHu|Q&pj3s8TX21e1~%XaA2bmp$14T zG!y(wX%Tf4#jF-8D2sj^<9(9gTgv?0Qd020>)JOFdzODv!WUw()`U8A>F#F5UM*h{{qF)hk>dX!hW2HJif80df7d5H*r=nDUG1mP9 zAJ9%Hn`8R=SgTPf@;4TPe8m=(pOi0@TXojHczg-1zg0r$at76!%cs+rNeYmC0R76v zEtaDKSKKN;E0_3GCvz0;^xjERnqj(kI~qJ+eS z0la>50Ekoe*Gm@R4R*QbS7^uHF?}~vOPlc)>M7r=V%iF6Ucq%2%~WpMW9pm}RzILP z+QH8Ey(Y{1Whr}>{;x-7y%6KKl0fjx^nVNsszMYe%5FGa;d@};?_1TF6bfkvLg82c z!WrNZ{}(Tz`G51eYl&{x0f&QpRv zLC9! zIx*0Y+Otu5Rg8#bsIFi2K_QL8?iUOSM`|$-Wbl$?Ve8G-{HKKOY))s5agpn>$)#pK z%+X|&O#(#Frf8pB9G$a3>ASMcD1>OXXm|O023Qhh^8XsGFGi}5A`rx|jhbsRKq0j& zfz&Nd#o~{QQB)94C@}b?QtrFy82&+X;pn66|FSkzPDtmU>l+`E{dOX%?nl-UfnVVa zlsFk^O&BSEI^pO$m#KBCR6q*ZgpW^`o=&iB>%|t^7_*5iNp|y5eAeXUn(nzWt(_q> z1v7Z)7`ycfYfF*x)3OToN&NpX0h^J3hR#gc-Hw}TkGH*fUH22f@cgeCQkeOSOi@@( ziEgsudCaE`H2O0CJXVGnw>z;~bthPHEVe3D+2;ZBH*v?uI+u{RM2o>PO~+@F%w z-`kLcQf0kgzz}qbWO-gWnLO~)-$3Lrf%rkIr#cG1m7NrBQ^kCNBo^SY;qyE{W!e>V zJkt}9E@bNvY(HPVx@Ty4Xf|;e)8>Dy*k;#HQgUk~IOM7%=~cbUp<2C%tXjN)yg7<@ z(dmz;ZP+pHdh(c?Cc^9Vc+Ts0FRv~O*Y(`!kxb3TV8wpi7Z>H;cC3_FkG@%%H}QVIp^g+ylJZPyN?%@zwx?WL2cR1t2*JZ zoWT0B`n4^W{EUL%{~+n~Msq&)f#yG_wQvZ^xS_-IS7Rkcc$7m#Ws#9=WO|I{HIzdaPcXvXem zA%SSOG|CHG;fQGii~HIvG#5O!S(WH$*4-n=v72M2UV(j6In9$q1RhUMh3R}6nNI9Y z*b%3?m%_6LWixd(j?3>TE)FGb+69F(sSB!j9079g<&(;F*wftEb? zr^CN&h}rz78O*3)8g>?RlX@4y7vA${W6ON-1a8r3lgJJORXmJct^_2F1u?hL5rb65bvH{oyF zn!d?)SUyt|^LSOJcf;R(9af7GGMi^MtTFVb;NcLftbkZidAV3{jWnJv0o0x!r5jHc zY#X~9vl@*r6p1-azN>%MlF#yLZ`FyqcxV!~7|AuaT^$m(yGhGeaCL>I^VeyoVc_9u zi9VE@jiv}Cxwx6vxm@urcKRh7ikA*G9`o-UExNh@TU2XW@hmNF;i#%%%zB=CEiRg0 z6F(k|4#-ogsQ7OLjBM9PQ%2Y0V!a3GP%EGqC_ub*x9a&nX>YA!Wtz3imeLv#A zCo^%Ct~#P@$*Ng=tq7xht)FhPeD)v++6y-uPf)3PtsZ zsJxr%4XMyC$;khjI9+8g@J7;85mbnGeOEmGYy{vRkw(f9TR)q6^Sx2F#^+*W4IX9= z%|cea>)doUd}Lm}k(thyrPI1@GjRtBJKh>b0|j2&XO+a~C(djzC@f|3_yej4eRziL z^#KV=viJ4Y_3-z*ifz)Vnc(D|5SUjNxb@9I_eaevR|wmxW;>ZH)2#A5wY|;=$XD!;8ZgDg*{;mKNx~&>8#;^aO20f_d!EfF7r3+x z1iup~S(m&X+}5GDgDn!KrTbCENJeNi-#Bmdpzco}ld4H2AiD3(IaYgvSMUd}Pxz@$ zJx^G6N@uye#1fVZ@gnBe?JqQ{0xh2ZBD+7s(h+wz&;(fV2=GE(OeyeEZbfwzi+NxH z4c7eeC({KZ(-+I9JFyHT=`KN2GVfX*Yo>Z1hG$>(n&0e7!c>Zy#^}!q-D>-LYPO2f z+__3M*dQKjqkCPHVKL4Tpnyjq34d}fkDtRKy=-+PD=8Ozx&@?e0kGC~}ssJ(Wv*iR-Xw z0uAQ3>J%+>)Sn9#pXCfeEs6ZE3e&r;0?*ZZ??PRTHuFjfvAqvhKl>c*hH5nkU_GY#I!WbjUh`tJ$5lPr zIgc&d@i%E{N|NtN%l-JzEEn^{fJ?KVXvt5DuUphFC@Xq&#_cN63NLQRSA~u{JN)pT z%}ZAjKXth6pM_dq;*fG^u)b!pTRIAPU1;B{wB{pjXsxZa3((!S_y%W?OOoJg@au;U zr83too+M!E`=LL7{-o{}_Qk5-vJQ6N<YCcfdJ6*iC1Y@%WHNvXE$$R=x1W(p`;uCgDDJqziJ0&@iNWDgVQ@` z&l;~Rfc~pMiwT`|`@|N*Z-*zV5ABsP5h3YKcFBc~jXXNNjLkYDxV1K8_u7zmC$CT= z)RW?NZ2qqHm+nQBockXmuR4yVKEwd6w`d3|8Nbj!TvLWSUr%MeVH;mC?G>h_Bjb(9P2Nkkz)sGp>tY<;$FX&WNADbg(3Vt2B! zqSR^^vAa=m++U~&2o9=q*z&kD|6aD*q*-J+4hQtyH>Zh`I&HjuJ_!j-Fdm!gpQ5iy zEb8>Vn4D1k5_H&apuW6r{(J_pkg2=rjJpvgHR}w6^BF8pu@rqrjR61c+i$vFJ0Bbk z*glTAmwl*2&hLMK`~@QiICqUkH^QdqZi}JH#214=q(Io898>nqo77rK;A`}@j&lq2 zJUkF1Lzd-vq^U4yZIB~Qm_X^vlaqQk`ovPVO?s09&qOFlXr;p5k|^tjH72I2_{$z; zQUn~0l;|?rnAV?bhLqc$tMyMsG!Z*Lw2O$~Y=jojnsGQ^!jf5u&LLP z1>46Z8x{k2UCrW=(ebbZXd6qV`;_wZ#YkcBlZTC3uL}=ij!pgxgrC1Ac$Qc2#hxB* z1J{GlQ+d+4<|7t(VFN-oN1i{FLB)R_p@MkLAygP*fc@Tu4o^b- zjOyx;XJ(%78@qECk)FEoJP?jGbBlr}@#YUg)!fWL9>S1&ob=YFK^J|IGZ8LVnAH{j z-`|`1fV;+1qjL5&>yPL{mYhu0Q&>tZhu^ODEO99xo!Xth-yj@>QD(-I_XwWz7Ufhr zw%-fX{+#bom$X_waDV4wk;W%VRf`7TAQfAx81WhI$VwHiGiM!A^lp&_jRyhY=3c>+(M7MSSXG3PV_y97-h#9Fb(Swy zD+peEeHC2y)BGo~QZ?Efjc0OQ?rybIo|y-4V4tHSb5+!J?} z&!+La$JeP%`DfR4*Slp`Q4Fi=u>lgVVYy4tRa6%_@J;%#?7Jb&ie-~J4#?CohnUGA zHcAt6SoGJK0j1KtBz0VJUWeg&hZ)>YhF`kuR%%B;uM5@j3N}NqWJ(i;H0oE(4kGPd zm&%|k@haibVt_K&{3Y4*<=&J8I(Q13&m+-#n?a+!$|Dr*1SX*Q2|Lb$2qrRz$>LltjQxYMG5HrO0u4a)8 z(r8R!<=>1p@eCEdYqTU3B}6%#cT}CIB%@CkKQLHspYxf1%6vb4w#cx3xQ47Tc|hfA z85OmDPPQj>4l6&Q#Y0a#J8&e&eE*j*eV%3Vg14b7{Wj5|%L9Vod@&dZl&eV7Fz@NJ zo~~7p_NxAhI+O!d@@;AsVvyI%6DRchn8KnnyZmPL_2gw|c~&95@r7-`pJoT%j>jPe z&Q}Xs7auMTKOS3F4MhiO}7xM~e5r=9l@LZ$6N7Z{p{?GhFuTQ0t$S=@ zhQ&IwxHP<0nl#^IMwXuw7ZdJ?$`qAw~ z5H$J2F~uNy|CuuZOSAm1;4bO&nI^&@|KsKd*<|htLwtvDrV|OajB09g@qqjO%%Cf= z`ok4QvNxyo{=K6)0AY~xz3Ci9>9!EPQCgWY>Q>ZZ)3dTG?HL1Jz>L7fY~FzQbm8Es z@neMd!t9$yQX=8l-USa@y-~410hzd<>cqTI} z_Rz8OcU<20Zc7$+^oVC(duursHx=wA(*FH-aJ9Bl5NpTj9|Cfb4@EktY)tN@Z>Iwm zzF0#FVz4J2NwXqQ#+|e$+H;pINv_t#|Az5IIApIicC#<@7-`~6!EK*ZI~M5^^ZAgD zobA7Skol$jx(u3g(S7?}h;G@wT99OoTxh@S>WIuMJ9v;rLI8+n_WO4SzmdoBd*7Qp0o_x{ zy4GbVWLW5GJTy%EM4vZ*|1gR!v-9%yH(U>?hN6cO+r&qm!rLn4cYBO5ytdv88gO!m zQ)xiy=d#rqa^I*LNVy+c>{(wFrKN&(WxY_`QWE1Tcruq=CQZ`l*QLhO9l>IodG++u zzUK60w)`v85d&=9_P(vLZuiUESdjy+393B5vp%1H*fmd6Kwqm^+C=S9aRv6XeY2#C zrWL{$la3;`91hxQ^;nf^vrd43wpO<12qXWbYKOq@mFBqi!TVIgfkY%Xj`bj|wjd`j z9Xk0RdN*dvkFuqZ6 zP2EmU3l7>B$gga8Mb4K(q~e*04ZzW=I@%$n8U zrgzKYA-Rk1lLkLoJkLW*qoBuqX*tAC=fIuUxc%A2Lx9CqYcNWr(`ldN!sP97^iEj^ zP;H|CV*9I1Au2frf=gz)M5Z0;AX+%;V(=z_j7xUX;48Q)?5|Jub-c6MZY$o*Q;>8% z)RT7119(OG0b?~JN6FaI-)dsjW+aiXi<5C`>d50V^r`l)4<`&2Xueq9V~~}h!+H57 z`xT)7Mz2cf!{W*C`r3BsT9I^2Hgjg@t*AH7(`vY_#||!z2F2NoMBPc)R6JMYqC3mW zs)fW;%3YNLJ+ic7oEMf>3$%;mEW$`n>A`+lz^mRHwd7oc*pE0#6$TxmcjWPtIxQeH z>*{0*r53m@f>_ExDS{Nd=8KC8xY{Ta-mH1fKv}rK6CDpRGF6lVX=cb*J3J`j;hi1+ zw+5$rl@}C)urQVQouHGT?ZND#zvW%Y!;bL|WIlEmLp;6P0l=q^TUX{C_C-s6kdxCi zE{X&`F3LE?(~b`S9`Av0*D0^X?h?c)b5LcUt1}#$j(Vx@$9VX7&xo@V0?kVB^)diJ z35b8(F&xfC*Ahu$LYS{8%J!cUm23bTy3)-y9MW>nKHS{PQrnh;f8~G=KPNnOXIUBnvd3j>S@0U2m~w0)h7xiP^(C_Km&(C;tAkUB>zMt#daz1^sI^>^rQ3VMJ2v`smcsoTFWmuAic|# zei9Pw@mRBXTrFy{H*Hl~w(7sB?|LtrXL&$Wb5=gSwWFePoQ+1^zc=ErdD*4Vi&<{O zd+8nIwA?sG@^x;wV5iIc5GBiZikMI%E39a!brOWql$Ze~0ON>bH&ALiOU+B^7?=9`|Ri#2y9< z(gDUPPjXe=3cvuB$sN%{PrmeH8$o@IpSIbRALQPuIIal< zJNNnEg8{G?t254;;|aC|lB)olE(7TAUyDNQSmn3wJKPU(Kj{?)KZLx#Cui2EEw*bD z`ZZ+tLx!2t(X9Z8|fsW5NRw|$r2M1_aS zqOv)%#_??m<>6|&Wy$@I>RSifMxIb;!IlIP* zJ=#aBlFj*kH+C21g^?nwzJN}-0@l+9yllnHY<=5`r}fcv23)wTNe(XQd9^bKIia#$ z&crOG`ylsOJZ5D=QSO-rTB7g-uKp}sH}IXFhDhO;Y3G!4&bkYB?E~QGxEyWX534cl zu;cSN&%&{*i*UrHfxkx;+J?!CTMrE$46uyO;{J}eCKZBR?GD@K)P{SxYYn!V8a3Bm z#8o{HO`qRF)QZ1(V~qcjd}ij-GY)_hgBh>i%)Lxb_g~xSy05l6YYXz58hNSOJ$$)(toZmy(KsQaEpq@+-S&fbB`TaSS-|JZ?VeLX8LhmkY9QP@RDLG^(}Q z)Pcp&v2ojAbQA%iCF>N2J6*p|Wyf5X-4WNA*kS}rm=m@E{V`UaAMp{WVZSryZw4WB z>E2}xj(53@W7A)sb6A`)>^a(-?aqg1RBNq7)hOb1$lyMCw?5Mq!$w0H0_lt5rG3@~ z?j=fevPgo4KJ-J`ZI11?5v3UYCqoQ-VMH4ZdY&g7VBK)9TkYj~`=)*Rg6PHus_-jP zzVULT_!B=VVwNi#jj8+WSceYjzr^+snuE=%fTVOcd|pCZ;CS7$j>YcH0oDSJo~LZm z;Y5LlFsLOW+3Y5<70kj80f}NywAgu?bM8E}pMJa|TVZgwRZnQE7sO#34;{=cryU%N z+thu66`Us4s68d^ESW2`)|YVV76uQUZlDhj3zDRMe;AHlMc@A3xs3s~we=kH%*Jd` znyoa+H5!nAlxzG%x-jgn7nP!O(d`1e?M*Zv$oTLXt@Xh2HOgX_-ve%1adY1fS~RN7 zkd!E!I!M5n2A)h*-*-iX%&kSjbRT=XweinVbJ?ATEz?(alrqH0Rt8O{3(O&yJ8zb= zpIj$qIVKhZw|ncG!bJKw_|w>pCWDg345NPy|HQKj7l-@TioUtI*COR{Qd<6zD|s^C zL@qnVw1sqNxMxn&v1V2-S{$@z+_BLf z@;+CoBc@(S5NN1q>-GnONP<~OW1;d8*5!$f1=<(I_`b1umWj)PIMKdWekO=Y8=i7z z`0ZV1({CR&s0eL3ww0vi%Inpau{jV`v)rY7yzsI9RW*- z(|E_yM0NxFy%B1SQ9Leopu5r0ST$Z`=>cc(w4^0SM3i`vFps<)iF$de1NGqA8GVcT z8+#5=Gyq#*ojlAR<2{Qmy{Z3$c!$mVbl03gFtXWSkz|=;B}1O{U5A(p(xbDsY7-l( zJMr6)?g)fCvES|9V{?1hbd&+po1)ikIbB!+YxuZWbwQf)aMv8KRvAE2nElr}@pRm^~tlrL*sK|QnlbUgQO!K#E)0*XTJe|zA@{bO>WqY*c z+`oUb(RcP>aRTD02|m5+#KYs~dJy=t7sAX8L%^ zm>A{DZU_({U5}V8-XuCWeI_!nR}V6c4&^S_AM#nf`^|*JtBSNXHhS;~1_?>E3=t$Ct6A7~)x zUaW7%|IR+Q{lNok3o+cixlvRAW`W@-xzH0I17^#8CvcoK$%@JpmIG0n2>e}ZK9(Eh z2Ts3cp=<=B8JnG|wE;l1V(q~jJMZo{V4NV_cTB&OI``&>9-}F}f5*LLR2$30Mn>q4 zuURAoq?%(q_-(gBsZKJ_zO1x90bNgnl-UEiY3}rr=X}bOWFdtbZ>)dcfEF-2gp2|} zb;l!4aB^0w=18I?a3(G_0AImZN9&4y5G$npAQ6}P>Y|eTLy-r0pbwV7gwydKyi(6{b##JlRxW>UUs3! z{#}ZJ1lac(@y9qiLdgvu?@ZuUnIz+5c^LzLm=ZrHaTz+_MzupWDrBf{_B?uVR}hsH zGUD;S8lMT+gBx4guU*7BVz`%}!0Rc(z^Xk*sE*gcEUiV$1tpqd3-Mv;jef#a?9!Y< zSBR^Iyss)P)*}}P!w%B?7jE(vjE-GCM}3zrWtUyv1L!%X7Rbw1!VsZt$m)#FJZ|m) zYh}x%9rX)VeluyHp5#a5ew4o2ug~7Eh%O4hfX6Fo-y!L+Tx4S7ck~h=b z?0%&)AT44?Qry>+rqwXEgVx8X(144TP=Him6C?&V-+=HLtE{3z4v zKbW3p10#uo5njaMtY3j^kNDdj$ATXx1T@DaWT?I(9U~VdepH=wg9kPan_bO{bD~tj z_fAlw$)1@E%?HQX4XH3fExKnhkeTvpDo~p89;@Qt}dN%Nwc%&h zmz>UA+8v=AYH&0wT0BuX8Q#`QSnPQI^)vrv0Wk)WFnd3$QDUdERf6u=^?gYYr@Ro; zea>B(mk6b^pOJ^exq5H4(h?fSS4_544z28xmufw3!)@+$m=culXP2yJ_S#wrwR)ul zY6p#HZn3Yc3TfM>H2vJk0)>b#~ zuKOa?dGzAdFgwx_<8$Sh*O?qv206_N3zGlfPDTtLLE-pY;I8b#&>ZzLv0BbP3ha0W#@@<@HUB(ZjA#6vh^FJFS^fq0#isHg{SJ}R^ zEi&Sf`Wqe|8id01wvXBXL(D3huM6(z8%1AiG8 z73GY9?~WIxlkCg+DVf&yjhHQ?k386y90eZbGua0783QMYx%elM3suH}gNg(B%gQkW z03ljW!eK|2Nc=gzF37-aZ)nZt_pk{1wJe{y*Y`%pUZPh;h&SObp~k;)9!&l2fAE4dTOL)-u6jy!p;MY!U>!<5;0Ckm(4+c`u`9%gErcIuJ>E% z^>J7Ry3L4!jnDr6So+;Ybqw)k>+!-R!1dcQcGD&zLEpohAQ9+)nu@JsRb@mQt4>mk zp~YVFt#kyNk9Z@ehX_4MHYri2@9T}E9M51zK(Eq5cor}+C4+c^yx;Wv0!YO-&<@j> zeE?>WVrT5gR^pXyt@wL>6)3N#i#J!qhnx#hv%e zkHZi5SV0VUeH<9;XTQZUn6I&FuncuEsNG5Ku|mK+*kER#W0%638GM@Z9HBLA0C3&F zO8kEP3jO^&ei|nAV4CadfP&triQDukCu!vm^E3a(B&C=Ld(K7kEXcK?`s~!0*m-W} zGWx<(nU%48(u2|57k=(mWCg8}Y+0X>vYI^$N(Jr_((K2v2m5R3J z1n68*6;i2rXFo#Na4J`ym&@mIz0xPsKd;+lN8{M^{1q=ygDsQS&O(bxhWZ%Lh`zA~_I`UP1Z%h%Yq1W73y4cyF-j#m& zgh?*@UORw6i$!qj+XHJ$950c+pb(&DzdX*NYbmRCoqS58We;5dw^sTViP<)%AeQb~2wbZ||7#xg zSK=xAr+_jA8K!n{x&RPt7AuL)R)==_#g<>aT9JCm~r8Bbgh&N$(L7mJ7DIsV3TqXQWoSq4twn1#C{-DhDxkj)D9p}LDFJAIP$L6@B+HTGXy%s9yc_SgPFr-#A zD%#)wS^!^@EK>*Zov|^IaZ`_9fmyI=pDqKxqgOX6t}v`%M011?7+`rF;l^bvU=;Vv zJ^o=OXN@-O(!=t6v#dfvH{1drb{4o7wiVh$0d#|l0Cr_(FYGHHZj-^aUL2f9=dWhh zH^NwTF1x}JDe!F$?NX!@DN!^o84v&?d)RlTocf?$s(O-*aid00h<}~+;xqvzEO!LZ zdNrQ2y)m|T$$ho*-Vo)BmeFa~L*c6?iu;+WtRaVo$316`(ey40z6InS_OrlWC{=ar zA5OR&9i7*Dm$7pVgBVKF*-h2whByfS&bXJQ3GGZu8@z@ah~9oKrN*lkel|2gvrlPq zYB|5Iiy{xM5j*SQLWl~Q^_U}>{N7|6LK0+&y^)w7^?nw>&Or?ncSQPLC#?V94oJd7{PR;BxE zSZg3tvn9rR?#F~ItYU^>anRF->G6m^?FZnAokLBvdQF=GkT>y^(V?$u?4SWU3U=`1 z?{qdeEHp0q(o~VmIgIGY9LX7K%WyQA(_5$#vL6I4%JU)qUavw-0V8D5J-8+k0|oMk z^V(@MSK>4}@`5C`7J4vClMnaUw&W?ztn$T~bXt8ALrpr@nMa72dm}3Q<+LOg=I=}x zwJMKDEI@fVxQg@Z9d&>H6x;a}|;)>9$A8sS8uVDyo zOq!W2B)+=PzM^sWir@o_K1P+E>R2u{r<}}rR);1zCOLOu&p#L&9=qN$cmjQP=4Jp< zxI$cxPhzfqDFROnYQe?PtFnGQz#gz$S}cTY= zQC}DBd$cl_jY(Gjemkw5OQ*%~+PmeD4^&wyCpm!EIyRUFU8np#?0oxJn9YLRxZXsT&tzCV&CCCiI! z%TUxk9B~8JRP%eQqwo99j(_QF^P3uE-t5l7h~|HJ7i#dlu4glh2u9?3HmqJaG$LF; z07RB)RUEnNO@w?o={DPg**<%KBQPTizFCHD9E{;y_zrpVk3=;VFc&eS8@3J>>{s*$ z1SyA6M`luD2YnGs(J(!!g5JOQ)D5+KaWfgnXv#;Tyt?;Tv=%x5|PoT>~?*?K6f}xA`TM zD-Lo=Rp!HYk=MW(rT(@uZ_%K$;3s&iZP=pw0T(+m$mk4(XvJ`flB8U*!e+Ov$s-H5 zP_vi~OYmv6ikfcTyzzPq@HKXNZj6dhCRC9n4|Xk*2D=fammHa8rG+mE<+7e3@(oFJ zxnfX;T)}tMW&o7;yS?bN_`Ov6J;-_2y?DbfZmO&!thZ9vk2`dx(XR*eu(1ee=JX$o zBkaYso496q@DHc&$eS?m=R6u`+}itF+HQkX&%3={l!i*_FG-gL4R1Dzx$!{c<38=@ zfn0Vg)6icB+NbTQ0@s&iyX`j1Pp0LVpY?M5oYuX+J*jr?UBur}CA`&a(?z`~tdX1O zhIQa-2gcgiBTq!vb9=zZZag4(l#Y$P%MF(9+bkGfnQZ}%J<^B5nsep7UA#-rf1w>= zBaN94@(bvN(5JT1@)(DEL%9 z6<7`&IVK2W_js@8~1$Ewk@1~fWRFAZ|pI^=#TUNs`BEm=X z>nND>r*;miEbw#pFN|W;51_KjqpT7Ud+RrMfztstLipBnwhD5Z@aB!HzYkP3al_#oL}cpcuMGqjxlJE7g~?}w_B@bBy>A{w(>NV1 zVS$8IJe_-LYhC-kCX7LpmKn*^920WGX2wgDhM3mR z@?YcK4;fU65Kj`Tl!kZxSX7i6{7n6H^JBHRF5hRlD{0Ao)^X3*1BMEnqctfo7EJ)- z34JmWPU$n_*->B1I={O$8CmxVOD^QCTLme8F7hiEP3bk#9`^`zC7D7b>h}n{4ouSP z7ZPHK8J}DaG;(A3?lB&0eX%lyk>#~= zV2qqwMojiR$t=8ywGv_%X`wE z&F`;Q;Z+78hq@FaxxcsxLoE{qrFrvCrCFIGzCIx|o96L70S$&Li$VEsCWD5d@)1|R zZRH7p>~b|p=?Vj?3$CxIU|WpbjsP;oLoNYABvQk9nm+enOlwdq&S|IQ$h^0NtEZOX4jhYqQf&a+7{r5LCp$3d|3VNrrUVY<<*5Zn5wRPQ><124n!2;CqUR2RLp zN|;raLw##%F#BS_S2lUykCS!g(moCydgTAWIX|vE^xMkS`xHrv&lFl2M-{#ilT)gst!{}P&R zT@5S5%>dcBdMJVos-hH$Q_xgGY#WHkH>*33J8i{*tJyi>lgolkIfUI z!cxjrCgdZJB1!Q^Q{P`S>ZgQyRo*m=(slj_0Q%1kL+3&9;L3iiLgODiP26V4q?`eF zpR$*YCHrgp>wDK|Ec8gPZFoTH52L+<>5>gf`Ol?twvOqk=4;nKlGU}U49{C|#y?%k zfh$b0t{VBx*N*sNP}?`D@>_cBXAYcOV&_PsvRI>pK}4HT)zI)($3rTjr-e3>=_5|g zlN#)e#xpcG7tg{r@)@E&^X+PXShV+zW{K{^Tg0xWi3Z9q7#7t+5Vv?uzK@baE?lFJ zGfM`&VcsXb8IPmi51%teXfqyg*Y=uBmMHV&h0};qj2G6|Tml*xG8UVQ2pWGz!;G6P zOpUOLHd>bs%anFH6b;5?@~ja2Ayr`yy5UdK8w%*>lj*z@1eRWz;CGjTXw)@PxWRz~ zpZa=XH;Eq4jZ?(W7FS0N@JG?@9t6a{c!1=A(quKg4hB(-kQ`w_MoyD85A#6llsdA? z53$cT(+@gt$mNSA7D;yb9)4cySxFPcRKu^3O_F(WgN9~V2C;+7lD=Ao=(E3_Vy*+OhPv2jo61XToXQ}3m*G+F7`cK!0+lm^nrU6c%U(kL0pK^XH z2|-)TZg%X6=Z(G8%k9ONb%5zb7o<4@Xa0jV15_VAPVn$85JAEjwnd4uG33+Jh~rP&^zB1K-D4b_hW8mhq(g z2B?Tpv9yxpy}QD`bUc3>nZa1zY>K(Kxq~{a09bE-RGEfVvKx)N7#Y2fthST1jEpmA zuz7c(Og86b$qxHF!m{OQ^?GlHrr|vq<>|HdT|P{vB|o;uBKAIyz|w5_i4m#h^!K$_ zSRmu)vLbOyaJljm+4c1PI9O327tk4nzP2E2z5u9ncwK7)vKyQSkjxj_FP8^iK3jipQ|i*i zON{?~!`r414k}W>-yhh~b&Mguj$vVd^;!}C#xwFm*Aui%z!@? z1ksG`Qr0X;#!J<+(lgCG!Gj*~_9k9YRv7)-g8n4BCTPQp<1JQ} ziFQorc?Ea-oKQh#yiewS5Jra>!_>G`l|tAP9DhDdgMU*BqP~Xh-aoi}y*8a^ z7lud4=EVp2vo<}j!=7OJFmB*t2tmtv7US17)IYh<+=(&`jWF>Lbok;CNECV$D(9)5 zBo65jy{$(ByD>5?nRj*)>?YH@A7H>I?!J=hjd)TDe6m(!h^aUd-^t zH@`Vc;!>Tuu&p;w7LA8?<)$tESo8YAN&^F@9DK@H!-dXFW}-9sESExSoCRT;l^qXw zBluU>gX_k(9f{LZ4s0@u$L&cEGuL07&x_p*W_4fAHEmYt#wHagrOCFcQ&4?+;}x)5 zPx?DXgjACxJ~<0S5D^}|z`inqcQBI1c~El12*l1nG@hW$p>K7`c_PZl)7_aqh)R*` z&ybsb`G8c_%oEQFs15aTWeNi-$P-b&segxT4{Y_M*naagJC@DaRF`JO^P4nUq>BG( zcI4%>tAS_0@VZi3^Jhz{hwGQ~1T2`CDkI6&A@Ggu^Crsr=8u#PCR2mb?{EI|UjRQq z3jm3M;$SI(|Btt~42o-8!iEDG5-h=j2X}XO*Py}OgA*h$IKkcBEx5b8LvVL@cYlYR zbMC$8-kk5pS8u&lyQtaB?7dd^>h-K%-MgQ?Ab5%L3Yb>x@t`bIJ>U1nz$g9ViBwRi z&(rCNzkeIkybsO&95w+ehc~C25a-7BFmTtjMcvf~5MV62=XunE@ib*r^@q+yzO36Hl{?hfhh;6BvQO>N z_d>xW9p9trH6!$eGk+4#(_qSAA$U`t)};mhg=K`_vz6++S3rvfZ4xD!s(HFt=*x%> znHcUH0BvJUO-N_53bHIhxF>>DI%l(?8uDa!b z9Sz?aQM$|ChCJ{dD?Nu2^SQ=F2g=3FF*|k-952w8iRYdjYS8qx> zUEYT7xtrt#m7=TB;8XSSa)ul*r=J~4FFuGNkzd-yjHf!@Ms~*=sY~9=zi}Ju6q*eK znPIa5HqZ_y5AS9~6t@{^Cl~wm>Cl)-ZvX1K(WffXt*)9(%rb8H)1O+6l=0(EaEn24 z?3LRqn@Wu{P3am?F-xCbVXJsJ7}T+B2v!1|feFG5I+R2!$BZ$&^q4QD;C?=@-b}Rm zf{N$i7XTLVVW)DdXoOCv;U*o{^c*c zI^ZS?MZL!}8_y$+=#>tMihFttpBg#}r4%G!Y}zR4gL{0$FEWQt7*OGPXJALKco;7s z_~~uUAL2u)LQFaoc!`96FZipl-vqoibj9_Bg~QJcAh&0E4iLuK;3;M}MeoBICDfnyTKE748(@9guYd&+qr3FUW25^;Csw}yrXesA1=SX&AD$xZ zZNgp5%wGJ|`x1M~;ff%dOrPh!#OlKQ^<6cyfl217f0Sxa=G8fJqK3Kn7j=w6r)mK>K)6`e4>qa!{ z5t$^z-nMa6{CKi&Cy2uK6`gb4awoWE2GGaPR>={*eWH5OqE;68Md9J0CiaTpW9qoi zP;|kxS53T{Sjw+5ZjTZCS8dQPYj{b_#3>Hd)jiV5EE&AzS^gXS(G9_S+Pxxh+mhF9z$^!|{E@`(82db*Sj*hY1YT*1!nfhBy)a%( zQ(f5m%BZ+;o;D)DO#Z2G7By~Y;F7C?`hQs-yd2;Pi<`5Gp2#^DnPT{ z=ww4~6GSYCdn?@{1AltZR=Sq_SUs@5&%#;OuF%<#zjP1D;`K`653ZJzPpwPiq#L(} zolJkb8Hjj9_?1pIi1cLY2#cRVpQh8QC$|=!Vw6PF`kl8St9cN-HNng11-T=i%9uj~ z?PuDI?Eaey;V)(ceouG#P67YpB<#-u>o=uQ|JBsLPgkk}A)$esbbng(U)TTp6xR>Z z!EtRiL$lo|iN9;Aj}M@_st4FX(OMwopf+fHF56pr4V{s80H~Qy zDG;5O|4nD}uMX8t0yu>@4wma+$8~gcK2}{R;-DRc{*7Yd-+FfkP=e)XZ=1*4a4nuv z;{Am~f2X+lg-q~BXztyL zi2tB<-UpHuvWfI=S*px1)aPNSsjz?OJT5joE&tNqkP}4e4E>h94PjsC12|smIT5^S z_;`etKCuTSUJLIRAt?#h@OVJE*8_MuG8@&ckRf6(;u(yW;zs2LYP=Y(L@&+nFW8XU zs4^yffY>1Jf4c&2?)3JTQ7tf|EGdX)~8M-M-jB`-8mp zfIo5nfAkHY0L$?+8W6=Bj#y#&2R*%l0?>h&llYviltFwf|6rJ<0MZQ=#0TPk_!`#( zml#AN5BUX*W3tNh^~m471S{{KcLqF&{NOO4YX@*L7XmdP=K<^tOP z^|`QKV;Cq=LZOP=fEkH!wqS$D`t@**sqWihei9ohHHlPeqzx){-{yeQMgopq8HqcN z*rRy+!$bQ91H$dC;QDlc%nIrcCJ;MLdX{L6;8>0nHJLnNcxfr%t_|b201TqEY6m$M zS0d*DMT7H(na-TL0J*&amcizb4oA;<~Q$FnnNt<*xG?Zv@Ik!rK}@8fMy zH+%gVB3O06`(oIN2*nN@b9nh^99<>({>L=o-5`}+2Bv!1EOU~V@eaf7uNHvu?DpPC zpQSlx7H`YNeq7|4FTNl-2)hsqL^SUh$p*JqM&-+Gmg)MEshS8sMX*9j4B-DS&rlfe z2|VO%8+EZ}6FE$bg>9gN3Vn7oKd=G)yG9C%YS}mZ!{D85=X{iG+HTrG~P`@!%FLIACKKIP*pJA*hfF4KQA z?605}I6cDN6$Kkl(rCiQf>?9nhHUf)@S%{1gTf|@xr#@vj{B2jxdI~k(+PSu+Pz^S z7(|%G+(TN5=a)bAblm2wdW5i{#WCV{EbtK8u>_yyF<%hmV zi623`i^fSn#~+tWJleg();A|~#^V`fSz^ikO7y7%$@3&*PXo<+44|ndK~R*yd{!4+ zg1j>EnCS6ixbmtL@-MDtM+coBx8hW@41hAtFWGMRc95AzX4BpNx%^+ zp5^^duCqI}?VoyYhB7UX%k zpSPS;D)pnCl5H~XJdroYrOfd+cO{V)FP5JvnecThrZMuvP&M{?_d$X-M~MD-ysgn6 z{-zY<)mWfZ(mxS@4ge?e;&g(AUe@4s-5F;(`&mb*3Qi)G5>e@>-SKg)d=V1cN~u^~ zVX5yRgZ0)27bcw9JLUQJ<6lc+)dK~T%8E1^^vB)!q*T@)d-?Q4LZ*;Wx*$)RT0)2r z1BXW6?dMRHuMqBApctcI%F^$rMJ!&E@>+i*jVAls<=ZTW@P^>&r4iZn!Rk*KJbKMG z+k@D6CAo&w8qH>$$182Ig#Oo7l91G^vD9LFpUIjG?RHLN`{M942#*Cz{JBtss}5&v zwuyAU0}?DJ5Ehgs4fX@32t0mtZLeIO5lTh%LJFN|Hd4s@h;9ZJACX9>#6*$Fi-Iq< z3@z_^Kq2hylp$$0ftMJ^hkWEJ#w!-n-5A~RAe%DG8A_r+0x^$B#Hu%@0M8KeR1RGu zpLZFIvZw6@^tX?i46(O6)rPdf4^)on&$*f(mmmjIT^9HvNF?&NH{R?rXvf)_EX$p3 z{UjTjvqkhPp6ZV#!yD3c(JWm&MIy;=uz1Viau?)Wa9_4mV8}D1shG|ZxES?&hxaS* zN(jo?7PT^EPe^JM?C!X@!wnHSgM~?yABX*;qR;g+f5iRjZl!G(MkH$E%~2Ca3j*x< zR$sbKJ*t9N9buJeZ6~UqnPRWgW!IhT(Q+LkzTz=hhrt03!(!k*mz>6u@nt^ms|sd| zF-df8$HsUnl>^P&hxKn3IwpICz%k1aQDkFvPlU8W$YeVo_<3i}mdv@U_O8IQmg1(3 z)aRJQp^SoQ#>=VwtI1a$J9WC3j_bRS$6XO7GnFaSh7w@yy zNtV$<1s%YqiXOur*j*EJo^*U;z*kF4pewE%^t3FQT>4xjQK&WXG@P#NZOGziYHS~u z=OZXY6s2KnY+bwZ?auByjpc4qzRTUOzTGSVKKIk;kw`Ri)@5lwagJ47YN zv^(^@!{Jio-k+6jek!a$0Ja?t&f1)HjHD{8IH?F|YOHt8S^7CZR2DN~=bPG$yT%|1Ps#!R=_>0J&gon9ZmHb?yZZlDv)BUdAPP20CEf=G~R<^TvY zRD+y+U|lh?zTEuiyD|UN!5QP2N~MwEr*f+3TYT14;|XY2>axz>$m=l?U+Y}r+HTz5 zaM=Y-WIBB8tMenZ60zc>*$H{mYD&#&_Vza|!Ku{JJeuQPUB&7Ze9gNo6NSFM+S4@m zn4`x{)s~{0XW0gWGh|yxc&}*~M$I|C%#;Xsdh+_Xgr+(Rm|a99JBfT!SrF$ zutcfu%>q@pU{-_Qg2)np3OKpBtBXY0s|=xuPmrV1nd2+|*VjCO$$<;wv*im!*Spm> zyto&P4(oiK1sdyoz_c?LL!RWhnus^j(@dKulEc8?EU>s9GXz#W(7FI1J;PyKXfhjVIN@Zg3IR}4-=~BD`JHY80 zE9$J@=x76B%@Pa$fHcPFg*+?IZUqsOKe%++C{-?WnEy7|&v#Kr2E`O@5W?rMaI-f> zp$Q|f*N<+UY4EC4^WbJ2dq3Y}HI@`3JB znl10~YV?0y1kiuhLRy2-25aXVXurZ{F|B)fF`^ zJaFCSI^kG~;Zt^qmZiGfgd}m=2~QO+6IN_fGrksvy*V|?lV1nxbLKLmYZnF^#C5K$ z@F`LKI$+_*r8@T0q}RZLAz2>cBbt|x^S2}G%S>j2ZuPdFxve6O!>r-so$4wB^0ds0 zb`SwybeoaSv{III?385sGqdvCe&!)@1Ggp0UQ6zttK@y7=Vo8@B9#N5yIlenNP_(F zZ)EZp>C>B@H9^~(*8)9plkgB6mBtxM7R6!m+srlA?1@))zOK8}rn#p0?^1n8mKtvZ zZ5cNf$h+(B+U9GXeZClZv-{agITd|ln^{V4Lt-gcoAOJW}ac_|&kn72z%a<7EoElJq#sANcHvnT*EjcFrX zUOlToV+`_;HY)g-pf)DC-8B3i(}>^h3E=GkL18tH6$2av4(Wby4|I#>rQ1gP-m>a+ zt>AZt6tWlC1P-%y&CR3R^BLI`25pz{3YNRLQRdU|N`68&!dhnV6S!+S8c&Vk_t@t# zp&D17SMOMK`cuCtVz4-m1$72MnA+kfF@Ch&pmD36<{+te;~McArLJ4D*PMLYpE$YW z^xo6VdGcC>VWp=UtD%afK}AmEKKH&-9@7XV{72?Is6wIUEndlbjuy<1?T1ILEKUnU zFyVA>U;)P){5-9a>i5ue8W>FGmSY}u}26y8%@{O8ERh#>B|b z$K#-OKP>M7ZODZ=G-!VJ*5lB>p6L@B#a*4}WRZr(+ZIyoAM-WFugu}?c1X^Hpm{mQ)Mo*Is zg$en!Oqvdmj>i8JUu_ZKmZ(|e7FBw6pqFj6R>6>wZ-uKUCCx}|6Dv(jmC!pBK? zU58e-VIoL-4fg8jb9VYO11xm($6Yqi5<-<+adtz%G}uG^aU!Q7Iq2lDPGgEPKKs50 z3-DSCw{hA>P-MoB;Wrq;3tlFLd%%CQX9|k(!M;<9+!yE;8ut%0$9K$Yl29GF^``aYnhP@QfzJWs?qS5%UG z*!(w(8Ey=eOmCn;+ThU@60e_Gmo<^W_!O4)?wCPe(QD0(={Bw;o!pw1sAdUw?n7$L z#;hfGe-{$yaHd?}8tyy(tQ$(=-$WGv$S4m8mXa`^11;`Oen{<{u`bzha9ljnV zQ0FE*!R16taezOS!TD`;sC5VwkhVXEjeiZ*$qweHwZ9wug*n}{divLfX>5phC zd^;%a=zY?rH~LQ+O6@aH-1K%R{AV&jAfZG8p(f5HC{f!VU$IRnRGt;V zaIbM%UtIVPr2CyB;P!=KHODImyN?}~*3xYYXOfHg5mXYwb!e_)Ao@9Sk&xtx%{oG@ z;sZn!l+#knXm%A3Iq+Axm%Kf-mw7%R^P-(RGpeErA8|xEavx^00N3O>6H-V zGpjGSCQX;BF4wXcoF)+m3Zqo9jyppMHOjbYRc_}H*N&2~Xv~@*%2@nt1!alw2Onjl zsYgb>s{;HUpr6ZZztqtHt^?BJbKp2k0BXguKYtwdmeDLr|7O(x{k*$jbrVRpABeg}#=)UiGp7V}lo z?+d0sg31Y9luFctOK-n-6%c*R@Ir{uXm%EIeO#vjs(UxgO_B>L8yUH#oaAiu#S4LG ziY-z58r_s2InTK7)AB%3ps}Wp?duIxDfYwCDVL$32WZ90^?z!|=(tf#S%ia1V9<{F z?fIqtDL>}L3HJWmDyx@bX9xbg5q2ltOgB}-XU6$2KTaQ)11_yW)Z7eicY|!`9Kd46 ze64}Oc{j-ACfnO`{m;^kr}EFtx@j7|mJWl5BQlGs3IiOd1NEOcJ}(TA2P&C`wV+TU zU2Ja~PZ#?S${#oSl1E#Y?6K$<20gg=Jg~XtJ;oZX%;a+>j+l*#!=`4qQ%)2{|7O5Wry6NN0X0yhPuLokO%}@jm z1X>)Dmn|r<)LIC=VK%=KZ!`WAQ1~Hfe<_N6&1!drzwR+zcQ_FKrr|{LeVciM&tmfv zzs9ga1ZlcNn$zKJy|A)?n|&sT{o>7K{y^t0j4f@jnYF!#x^Sh9MUqTS#|ozTlEW0` z(S%D{;Y8T%Jw=oiiD!-`h>lsp8p|7{nvd%Z5{bC}-CBopx-;7lFM8aMaNb;ahwDda z5V2xj1iPIHl41(828LKPd3Kgqw5ptO91!Fp*=fgqjt~~Qm|#y9&S5a<$E~x|PLPbJ z_Yr4Ok^0TH(+>fRV510>VK7r~lnFsISgVsQ-QHHw+~yR{N~YW9!P-~@ilrWP`&IsA z{J9&fW{*BxTf=j(uK953PB*sE)w7s6ZrGQ!R5*}Cm%Q8Xe#vIWV%dXWEPtKl=#X~t zJToRj5yU@B7TFMMGJFM z7u3dfNNx`$%7a+6H*FfUTQtj@x#5zvD0+u!B)O5oQ>7VQi*R{bXm2G2UAaeZs=oh6 z{sQ{!942yDaAO-O7<9q$Gw5|I&cV}|J#L7z90(C%f`nI^Ro99Yin5uN3YSXRRRIav zOzfnpmEr{^QPsJ!RoBGdf?(0{`L05qfj=*o77~fV1DztbZx-%Tt{bG6(=sc z2&d3)&iLPM`VmW`+72C`QAKMiO{zu9M5DB!05NE~;saTD`$)La=8&$wP5DWSdd25K z1qD|{Du03HyMF)a)G<-GoJ@f*K91Ijjr-#gCWuc8Ap>LMCNsx*RON!r?g>M@fqme{ z2}zK7!4|JkwvJqPvq-h(oz~C|j%3%PNdRos{ME;<^;~7LL0Np*P3(#1nfQS*#F7YU zpZmgC-+qXj2LqnZq}$X~H@P&Z#M+_&_G(iI#|=fP`z2v@bx!fvQhrIvkN5;Qt01N- zNJv;8H(vN+&FhX3jc0@S_HhkAXU!x22@22rXo@tI@i!;bO3V^G7TVyF?ohN~#I1n1 zS4e-Iq?G^yh=_Bb>-tXm^WQmDp*%{pRI6?ev!_#5E*H*tb=h?1Hc%^HVS*nmDy7PM zFK8zQ-q?T6@cP1QQ5@*CA`Gsz6+uit8xns-#qIk1iL`w#yP&3QujWv$k|2;XfyU;6 zy}VfM>Ri#8#lA3!o{b1Os&=i~jh0m2d=UegwpLoi*jNPXhijc&rGntF)KFDSsowF| zg9Us3>TsrXSvrEjk35o#+qLkouCciTjrDwU=WE*m7v8U2Y|<$Tv=RF}1=_8qc_6{S zJor}{-nz+>q=#ac@k3^^4nKa^}q`iJWTM#oRJwQAg^~1 z9#6@ld%EYnu~eVk-%a4phJg5KezxL9zcitbCym$j(uyXThZ50-!4jjtL(Z^zSjG|_ z-jPPFgHV04gfllzJ=n8TbJho! zT!}^;QNoi(5+7-TH@nlre44p~d$3pxj}1Vk_;Dnc-pJzm2rTOj+Ap?PhyGo+J(3_n zBVny^uAXO#9UK+%a#N5r9t%*lv0poC=?JVYTD9Uhbt>5-E9nCwzEqklzkT~2TPfkD z_jd!!YXk_`HzQ8*dzq9nUXX>4b~e=w!GVDyAmAQpHn-79zD_<6SWe*tMufChLaIx_ z@QU6Mbn=J^vR94=8FApTyeLi%3N0OtGI(&sWW(L0JLQDY8YQFY%VcS@OCHQxBHiET z`#W&T4dd1o@)z-NX#f$C@+U*>@$?Pma^m!Ia&lpm5&DFLgewnSrmXf)h`Qcss7RP^ zO(`TeMIF4LjG8;@auZLecRj?a-livblLWO1j{;YRT;f^>h~X1c{G5!#S>v1`s!Mqr z)~dA4&G2H|zAp{9GTpfT4S5m(%Dv-RZNJQr7aKAzAt6oZTzK}%SuHb~^0a*&C*!z{ ztQHdqzvRNk+#>-+6QilN9}wr>j8Mu_z&YUCxp z{|RnsehAt&Vfa_y&%edLLUI5C#>U6#P7ma&{FE95g)UA^PR~$U?RCK=(|7>^P%QIr z43SsN<)EF$slU7C)cqYLJ(R@Uy&ILpQjdoKalGloR)h!mSKcYI$7Lnbrgq5`AmPy7 zdrS}Z6JS6=io&rZ37w0B8vP4mTLI8SU?^wMn*;rcpaywyAk8k83#x*q#T7@{w~;`z z4A%3N%GM-3RUn12hfkTdCeeRf3u=(4m0vU6#TcjuYN*i!+z4A!iKcCE=EKoc_QbZj z4!C4%DP$DgH~yVS5ud6rnpA(knxM6h8hTNpAM>ScEyx9Fzt1#XQ&FB91v*yQq6mIs z(!624uP~npp`<%`TuAkGj8q0;kNA@+=;RDCU8d?u!GxV1$13Vi{|h!w_@PR}uK7>T zHbC9_U;w_0Z(1aWO~|R351JQ3KCk>8YR1}EwDsQ^5vbv%B98|)MW&&V2(qar!l|E8 zZe{QL!{?#G0}=ja8YNz@8m5{N51i%y!YJ?5S1N1aej-z5clc`vcB(K%{!_xH8|{Ce zQLx^xV1Pbn;-~C?DIN_C%sQuvo_>5@VSMgOu=?Mj-#r~ZthlP#wavO|av*l9ul~}7 zJJ~FP1f|$z@Yh@{p;5$H0Km?EjC-R zJeu)N3@w9#XZAlI&Uz?pEVp9gMW24gYdtv_91T6tBlRt_oe$uY3^vg$%kA1JG%ZfV zvI?hQKq&p{&mG^&(IPIVn6$4F8{7wqA2@Vf7oM;s8Kj7Qel7qKQ>P{rz`I8~ApRPV z8w3I~H~O3*{vyu0&~ozQDIEA|XC4|#>JH7{!}VRg194o)Ni`SHwhX(%6C!*rE*MW$ z+WdHEJ{%Zqjs8-yC%z0N9UBt?Mgtg?%cM_}YJp3RmRgYw_a7!Sf`d_C4m}l{>}0m{ zqbe%#HM2?bw(Z*PRYIff% zoJh<&mV@uqOlo=e7VJ$(1J@-Y{MPun(#MWir?U5WXLr)R&#ltFEP7odv$<7e3X6-u>WKnX52WMrndsFLczm z@57xJZOueX4f7)~DkgZ)Vf=V#Hj<_YWSv#3pw$j5A*ZP)g%?HOcQSKE#K-Pw@0@`w z{`rlTNp{xg1l3(=mBjrTgvY!(1ZmF1@>kiw;t8fuv+$g-HG-Fk)9Mb{+0{irhX*A^ z^s&5E5hth=Mm3(rRM-^ zYe=Q%D-eQE5}f47jq0-PciFCz2C0x~jmX&qX9bLEA=>dXHjgX8sFGtrE5!@;4!>abHlU~D*ICK;)}3&9tCLnJSnN5>aiidlWDbv= z$#sp+u5JR#12E#?R2Ul@hQSmLu%db9w)k<>)A9c0O*v$ngW#6uVCu!W42kLLJu3Bv zZ!o*~!ivM1$s6iggKFfu$GL?G6UiW#R1+SoJb^;z(7gffsE=a-8ZN3m**mX}GbihV z=1&87c=U{_@1Y#99Z<W#?mfMc+2{XF?-Kop!(6I)sZ+Mct#%(Xk%jedi~b900xk ziA8L90?FDNrHCkPCyubxic(xT7YI zoa$^xl)An^k>-1uUzLr@;HLdKbuqL=3kDO#xB<1~iZ@~!t3%Ypcgl-ghuDkOq0$EW ztK>TW_(4hueC)2VaZ1Yx)6mSCpE^%JqZ5Yq?v4pse2VI-IH$J{>h>U7HUd&xs%iK3 zk|q_NiD>*SdjN5*Hi)EnYpTLs#E|=()!eR9UYtQM%eOWy$Twh!iw!^p4I{5C2|RpS zxg!31&%4m6_Ej4CMNok!Hcn`~H&+9kiN=7lT#;(m;3rb~ZiN5`+qQuL0cao@8JWu6 z*VjlRL}1HlnDp^oKJw4K#`6lM_@PA zu#I-1P7hrsmXBtcFkJ#zMx?IF_oBNTQ1sPZ&6&+1ohGaEmJ{u)DO2o1`aJjI(z&m~ zmsT^r?iyJCb2S33RWLyBXQnMIkYgqz3K>PUPi-$(ob>hzYl-mn1BstmE)?h4;KIO9U`|M?g&P@K)jQjql>PUkh!` z2BS4uR1d1oXG(FzhsP9~q6E0qq%8{usdH!~U+~uzqw`oEHk`{ge^%K<#DVZ|2&WYx zGn8h~B;S5sTO9f<^JFf^8|FQq(jEkQH7Wx|#r_7WI}dDeyjb<2TY4>}yHnZ3)<_cc?Nk3gbP{zv z@HbMpihdL6a2er*<>jVNNtk2R0shZpLz16X_sn2SVJbMOG}e{ar622%UMwBtYY~Hb z4ts95uD}4&G63@6@w#-18r7BQ-7;08=T|*L-Ep78GaUnS|E>YUOlt!ihuVhKRIKN| zT89tY#m}wSV?xz&;o`5zgx!#3WJB^9Tn)h;CEPoWm7I*4G;nBM4DpeG7hhAmS3p@U zp+m7YkmoT~`WfXe2sxZIHYy7LdTn(p&WP?9R6Aktho3?U0Kh8tX0`}KTh34peS&F| zXj9V$7fN(bcO7v>C{g9@%~Q|OsB;7(b=9!_O!+bSVDiX|aIZ-7o@s3zN!vvC`&zrg zQ6NNW{>b2r7f3g)09#Sq4rw9Z$mXCtN+!v{NO`%gRHzqM zi1sr$8U7E6uL$$r5=&5Vkm(`xL-|Dqj6SNWs-g(enf?VYeo!f;p-=5a5UW{ z)SrGA?NVqLj7rowB-ForHE1wTfv2>mh;RNL;g&_!AmB5*WQBFKyzK`J}grgL(=wjF1MG&QgmcfAh z$}4hT3Mc-UFfj6Yk60SE8+2S#+_Z>@=BZ$i%nH58p z@3&2ql^bWCdqD4#^$LtFSub*3bVm$uEa(>)0FTA<64+|5?c3n(BW6p&Oz$|D4 zzry150lhvfaK^Te7V*?Tt>%lDtEFwYOU?0|uQ|taN7mQlj)8HkzCJRv%TuGB$sz(! z%m}RhX*$HsKx@-@DF}+cFt8g{%Ow<{<4v08;hr4s!`DOMByM{s!mW5eTC|S0*8+>G zlTkWetBr_th!>p1I8rH>!q4WNXD?23Id7Whi_W6vS82=1`QV5n&0tSXGV5F`p+Mt?W z^~O__m2Ix$CQnA-8kZuM>pm2Ev<%h96;JYXWZ@>D2Te37?Th%Qww07 znIE$qrt}vRYquf9CQABPet5t0mGR5dL2-_sQ&$g3`_C!%`LYSob*jN3ZE9?BF;rsp zN-4xl($&!p-%ujogLXPnJ?Q(>ub&y4HSZ#|jXhpp-Ou)1nTzBKH1WBtgbX&LB*r?Yc?rA(F) z4*%_GOu?CMUGFpE_wI`B8wX%o7Cb~~GU(OL$!tKzr=>b+msHtf*j9ILjm@fLCpewm zemm<=6TGGm%TAW7Q-`k_xlGCr)^Qx*s2X6queF8+FUgU^#?cUOn{Nz>o@Q;}HP|aV zuR!e6chg0KTy!#$W*-fmPqCzunsJYnX_&o*|E>X?TgtjWIMe8!k4@Q9D(`Ouz={O!>RGL_~I zn<}xHxadao;NEB*dS-T(6HDXbJXp&(-mCjSJ8T|)>@w!4il&@Nn72EAkVBmzH z5&F_7KM;t{$U*x{Bf#N^eGR1?J)_^6TI4mh6V5=1-aU^I4P-!s%%jw}OtkfHwB3BO zMXCHANr#K&G}9gwiJzM(-Nd;~b9#4BfOs#;ig|g&yQ`ZxkCve3A1jQnxxe*p^oNju z$O@W(1dBzeuyxTd1PI5EHOo=_1pek)G>*#;W=U$WLhq3_ZipRcBG)4s>h`0P zr#-EpQFw@93x48pBnfymw%_qL}{sbcW1 zuD6Vc=LwSqeqLE0I*EU*dH7W!qwsxocv%b5ECHn240UoUkSwYqrVMfM7z4|CcN#yI zjj8{bjM}V#si!AY3rLBK4t`twSY^cSeyIHnx?{NZx5pQ6>l#f0>^=DKR8e$zT(0YE zL$k4z;E#faF$7S|E4UwJRDV<*F$;cAxSnO*m2teULm33*4`%?amky=fGZsI1RFi=4l_RzQO(TOyN4SRhJ zNf4G%tFC1@C)hjhrkz|vS^e>1ABSGn5A;&AhqDgny4be};os#OKVbNiMBB8>h4=Ta zKAtKrTcq7O!lDte$=_FD)=Un%^aI_o2^Ax%@Vo5g#a-G~_Uj@iEz-oDWH}>9MTjUl z!LPoN*X)2+YVn$*xpwxF%EUlPb^fyBolE@D;Jf;$LHMUH&I3Q*tvGG6yDp}c$7C@N z@uRqHY!yO&PnZ_{m~}Nm8;xuTVM(*fp6m|d+26G9S5000%kR27vEc+J?F;a zb8oeAa!`M#jJe=%o6vrvVg@*xdvp@5Ffwh z;@o6b^rAh*RGNH)Vte~(qh!Envx1Kr)sO(^GS(bU)rsgWd&oQ_P71 zZ-F&?ueEc9yq(vZMxlaSf0|?~;EbO)U%S~gAC7Z!V4rv#Z8#S7g6&$zm93(00C_7G zRs3Hk8s`H8Ty=0S>xCHa^zqyJl#;ZG?qCe^wera*usbqpOyqK1d=}O65w+?s^^jY~ z-cYGjGCgGjM=8S$#0|hGW*Qx2Q7G0e$w$bd1g5Yd+0ioEpZd4Sgb@~`L?wDrPs8jn zWJT7=u!u(q`gkI1na}bb;kp^J9R3bR2I^(nQvg=>nRJ(7ojv1$+PrRg^%|e=bSli` zQ;rk;jb8}H`2w+ zPD%S<3|h0_0tp*04dKwA>EJrFlujdA0jX-gVhi4Zt;r2|W5@%2qbTp+dvovtKY#v+Eyx)Y^L|tC&Dq0~&gWW9 z;L+?$oC)MD$2$<5cF=*K8z2^>M6vc;t5u ztJA@50|OgJ`Hek7!rUl|+S^}`GWyG9(Y2(NeT$y;-YPycB6#qn*B<3q*C`asm%p(# zcv?SRT>C;$-h$e5UEmgLuIBbAwcFhz3CLF@5A8b7fsyDYHAcucUKG>ds5Ymjyh!j* zP3-xeo%~@oimbk6>HUR_IB@lSvvnWm=4Z|E#$xr8zKP+5K}o_1G;)ZgojA1cPcwY` z-2JRCnVvu3*WdlanXQ3;&VtvryeQ{#_$?O>i@}HUe}9gmzNNZhT{vI2rbX!=QeQVD~^6!_79c*zFg#VVnvhJ4EHHM zA|hZw=?Ho{osjNFVd+_~V@aA596TPNcCM;_Kta6$+po43P=p*!t-;@!XFX;z;A~C9 z`tkmKk_d0Bi!+neqIT5bF%$?B?+YgizwXn$mcxDrCzL9`UN@bAgcK%ArZU-YM=Z~lR&1w_Y#gwxgoTC8x zv3{w-vqn^c3rs|-5GuUXPuApH)OTP7Qx-7-#!J#XK&@_P`Qd13Q@YDqeoee^>Ky(O zCVx|!`Y^v8vzYtmU>e7F3FrHFkKWtkS&7f==L0+pLkNS?k-|^=gEY^Bj3PS;T1+_6 z+mt|B(rjg2Ym@HyvHq7tAjr~5X~9C@Emm&TvT0{d-Bw6($Jc(uahpIbF3yL$Bg`F2 zs(3?`bdUuBqf#1-b4{F-^qH^0Cc$z#MSiB{nUh@6swX&+?~6ec*Yo{N6=G3Df4!`9jCnxUdd_57n)Iv|9<~S8Mc9QqogPEY$MJ%KCv%MfFZUJ6a>%G|HLJA0kczCn2$ULv|stVqZRLt{0+jR`!8M zMU?rITP~La8pI}C-9oj{kxu>7YxH&uIPwncQ+SBs#IL+0GjQaceaf$J%t<>G z$hIk-aGAPD*6&u=%UrTKCRa0KC$5QfKo4PJ);bPpHQp8^em%HBzBpV&>{vbuO84Yg zyXh+!>kn}h6cpC!Y|zj#Hcp}Aov>JJ_6-{m4Z<9MixaX3btiB!vKpxB@-*WJe{?iw z>~nW%Omx?RhohloNq9enYS{-Y)_@zigA+H^{gn||fkJ~S`0?Q9P`SE| zeoiU#frh%`K5Hv(5FcB^X$o!V6qqMpeRUwniD8N#QN2mHHfpD*7rv{Z48NsFNkysL zIsQuD(1d(JkzozTM4i5u{LuTV^Uvs0oIAg}H>WoM=a*OPD22PD$(odWI9Xt8V-r!F zJHb0;Rg>=z<`w>snwk6yy;Mf}7 zL*5|Mp?)8S7~FEcqrd;7n;91&y%*;CPb`HWLVGsy%2yBopUh7M?EPVsJsl#>vUL$T zHw2#zjLU%oU3HmasGpH0boMFwm#_KuLsJ{XDP6ifxPj!je=ewY(*7e}6jFbY^xt=XaPaIIV30 zsM%q{Sdv{c3(zVnvXaf0m`@$^0_@@x>?WDzdfQMAj#1x3?Tyu25aJFQ2`R_!NU2lh zEbbONmRyXoX}{=a1!xL>$5DCwcZVpz37w;_N8UfC71RN-{}QPr0TgO2xkFyL{}t#u z6LqJrljGzl|3MI&c-ityQjV10S;dG@+GHHLw=-H~95ZY3`C9nWWT}>B!C2mlX|YCv zIWWk%u3>j(Rz<9!&e4uS{Pg6(TMgeLDmv*t29gjP0|iRFJN-lca6FrO_eo|o0ltT@ zKsc*F--W2h%=A;en(9?Bh~Tc*7SkI->oMelWViprzk(;f5c5~KlGeiregzFU{Hind zm$BdRyDxZxk|=~!Gt^qmp7Z#ISpEi|$(7Dpvn6T}(9x-eakvA*`9CJhU>bmmW%*Z@ z=R^AoYs3q=tRsmAILM>WW%HCu6s4*sb2X)^lR4awn7naC4F-2K<2dfIZqev-JeYjj>Qx2u z*JI(v=}+-;6Kz-58 zxU&=-SGevi8?ZwDt^q~Dap}^VQ0g-8sN5vXr1cGoc_O*)zcRhQIy;L<8WWdf(hHSb zzWCwgDG1Q>FdptbKCo>;Q#LlO-Of~|V^PTXC`(9{mP&Ah`LcL0^|`sAK#-beXW1Z; zKm9@%v3^}~?RrGm|Do$Efbv?lW)nh!1-B5~9RfjuySqCCcX#*T?(XjH65QS0-Q9Vc zWB1&;|Eu~c1%#qz&z`l`Ot0>qxU@)u;wyj%p{Z>ek*3LXLs--n9bHXfc^(RRep+TK zuwCzHy)tZP&Llp78s9F0S5D!C+M{&sAY|`+UZ+%}SvDiKvOg<-m@EMd?HdIM&{WP> zZ>`p(OjH>BvlTU=Erj?1&4PpvppDGqNn~{X6)X6WC)UFO9p-&U+p0oh9YC%XO@KFS5J*mkW^yj%^H zN3A}qd~5pp^N7jetLN|Angx0Z7z~y$fbOO{`Rp(7!||o?!E*n=bhz}u$%PxQRjolt zXH{ITZd)D#^)mIcg%t7L;6_ATs2iF_!$)#T($!9sM%@cp~YyxaW(<`dzAQ#2LnDCUugWL6-VNbOEL5hQ-<%YwMU$&kQ!oGOOZ+i&>c~KILLJ$$mHNMjED)x>hK=-#DQd;HrG%crbTQ2I z6-x%nah%5T3^_BUq(m-OEz1A4zwbq9p7&L;aoffn?=j27agB;AxIp#yOgrGG1zXX%aQGwj6?->Is&o&!}?A3x_nXDZ3V7GHcPcvmDfkB;2TCRrSR zfY7iPsf!I^n^cD_d7C%TvE9}=oC7Kvh}w(I0+VUn)MGH4$u1mg(LUw|+@{4`w0LCt z3Kwl<=Tj5&u~p+S4#Qcb9z>=A@rzek>nhIX^K-+cMXH*^_D?r}$nrOl5s1fj?iCS` z;kti4OZy;RIMGI*>Eo`(eR*9KU);#?8GqP_!BT!ITz_hz4zs%GM|1o2n?$N*jX}ry z{yQ6+AfGM*&22l_XxpUX>r`;u(@ivyY5TfwjEE=}X8!Q8HUy#5WXS25?1aq%L0m%7_@7(dqii=_py)PCZg#%$~+{ZZKdNO!+@fBshZ z1&>cjM~6S&wV7}5)c68>UtU|wy)DdcFqclH+FZDg#HO?u!M7BN{ThCm2)-T~SRiDn z+Tf_r*c$0C2wPb$BfR-A%?V2efP>mS9n%xf)4x~vdloOL4vEU{9bzH`w)fk|FMkmE zwDt*%R|++xS+O-~TWV`}L1{)r+BDzFGwwr*g5V>s9@SC#_$4q!+Xi4uQK{6q{HPZC@xM09qqjKQan(^BYnq$;$)Vru zI8r+ve?h?_)f!`3&nS;vq*j$jpD7{plP}r&7-)}56)B&cn}aV)5F?2ota=H>$t1yu zyN2Akj){T#jLaExSu9s~( zSzYRcQ6sgZOt$Sye&qjqS=S3HFz9x8K&+k;K_t98B}9-xswI$d5T6nPTWBl{el=o_qJjF7k zh`SWMYwVa(HJNb}0Pi=Q5p-|R+sjEYqTHSl2mIyg%-sp;`rWHDUa5}%f^>|}&B4ZV z4%aiY!B{%X*EV@nn9v)iPA}9ys_HiakSivks=tZ1iPiOgaA(JGfs$nd9+jsB;>6h= z?$hk=F9`Hd+nZG9G5V2Z`(%9R5RUj^PYF+X(s;1hLx08O__q;i?dVbwltu{hTVB#Z z)#O2tvj{k=4C^W7-Nesar6@P${vR_^_1dWn6QhcwsGz zd?yw3FUL{E;ccbE{Tljp0TJ1imCnyH1!C{xdo^&!(S^R0ruX%>gV=c^>h0%<`W8D! zeAnR@?At!&3g|%71J3ftVR~|0-h;Au48c0HdF-&Wub%tE=5>FR4QMXknw9c9?g?z% z7a_cfjAy^HoiC+8LOnAA{~xyOYg*t8YP7-O^pW(fy&NyIuHJuhpyNG40j6V~b~-C2 zqC+?6XQ949s6sDDN5=-RTcGN!ZNodDajWkvhrbTE!fE)+Tdev0;s(yiTnr~wtcucX9`-75P z(*`5Qe;)B?p!`ud{;89kRG@u{p&RBC-jL53jzEt&nb}s{-+9;a;)=;+wIHz9&|E(# z6jM2?$$xS_(-EH)rSzgQY>(0K>Gj(UPt0S}&hfu^xk--=wVYy5cBZpBUAAvclH!md zss4OAvc&pz*+^`L91Olw;t)+4)kR44)frR4LrvV#aCJ&`@_+K%-^U|P35f6-vF$+i z3mP3sjRs+>-e==QGn3VGYPZ$9JQv9hOmpbjMQRimFpjpSJH2s=eJY6$q%YfDRU*@P z>cH}?db6&HOACvD3~77ugcqA5=ieO@YG^+$EPX0u3N@2tU`ZZLn9KdxWE2y{?s zY3!k0FF!m~%g??YOaKOu!*DTcW`u_aK#{CnFIII|7~oEQX=P}p{rjZ)pLdXfySA?W zHH+iRub5S8AAh$rU5#kgEUwAfik-NSxwY4_nd|>AC-{RRVbjdl_Mk42KS}3*;i!L{ zFp*FI*PUd(W94apzpX2Yvft&y%iG(9LxuYuB9qkQO@7B5HkrasbjfOR&$>H z)Efec2}CzI#o{19BVzG3XrcT!lp~U4%tS3DA&&@2&`VluB31CV%>}5?-4%-}UVDB^-jge}NWYL8r@?!xF!l z;l}7QDlEAz`$QcMf%G1Pl-nRIM1p&+^GiH0+VAOkm|UX5I=q(_x#xETJ-PUVG-@tF zQlM@)ko9~{MT|6zUJbT0g78B$_Vo=FgRTS`e-HC)13a2!-RDG&<>%F}oP#8Q* zXdm%Lg(IXm&zC*3;)4J~SK1|hJBdbl1eEt#m)AsxJw}X!gFAx5DuBf&B<}5cX1u{jgm{s|cT^==);K?S>-Vo*MT;T!)zpNHHR zbq|galnq8iv8!gsuHRA`avmQF9C%U^`|??5H#+obZywzUzDyQ^1>gY~;S}xa0taJG zP+_L|4)OZAUwpjY0JEJ_d4tC92@wa1e>n19_44a%4a7HcQ!JOBe}#rC4ri-TgC1B>z-~r^imCMtEnvpw)KVAz+T9>J)$j6) zUC8dF`ZsPyJmc%4N!?AimwCl&dj!So$phG065*=8VcxH4zp*UV{ z9GfZi~F%IbJ3j5c`N>xy7ZC?d;_w;D?jjO1$MstElEmBYbD9nVJo==0+ezNT$-Ft&#`&0+QLS>C z3lf0S7>JJkdx6>fJxUDC8+^qH81|_odTwl6_NTNbRH0e`7pcn@%D%^3sEJJJf1qR{ z^+@(_Gju}9%S~EgRIeRtN|PBK=-#w>tmTUUMmJwZMqC{>iP{PP5cR* zi0ETbfgp-k(#stb`^vUtnHi;O{az4$uXu)RKf>NzE3}D8an8HC*ERTPJ8NX)X76<# zFg#+a{SMHx)fOh^+>LO2 z{dni$<@~aN5fSjepwT>%PlPxSxo*AImxQGw+K4 zbVZBUXV)qnBBz^!=p42(=zsvBxypNN2Zz|6snMP-TgA|4aSRT>Rk!6p`4V(g12v`< z4{9VM5V74G0p-Sl&7ABKL}djtf7nH6c_{bRTeF`BvF31CB^O%7Dz#)NS1I67GS>aR zJ+tI0(5s*0@kZO`t3IiW9{>I=bjYc_3h?z&I1-!Oy{VK>Y7m z#|5RbQn9rt?=-$X_<7puZLnO4MizV9(~HRN-I7vL?V&2_kn1zS%M?U+ujeh z$CF23-T9u($>xy4tv2=N#FUbSjZJta=Jf7JVxTg8(YpS+o3lE&yVzIycKbp3fLY>? zW%fpCn;>yziXr#{5*r~{Wt)C-#?#MuczEZ7=IAsAKQ#chb9do%n^qSK&)O-rKW?W0 z1Xm@!C|_n24YtJZ(qnY4j@30y?4RqX-^a_j*l-s@33RE;o6AN^e}gH1YVR+bhv3KODRG_;I|mG?}P>pBM-hG8iy8%6-R5^5gbk{^ORo z$jKN%?RL)Cp`o_1H-FhQ;*9m`?}-5f9Ydj8quE?S3QEeg18HILn&&h&D@;0j@mp5A zr|KoxbCc;z2xfbepqfL5EnU)os12MMxMKi4*umMk@K&Zk*bzPxUyW5-Q%5IwW8;z- z`QdXS6)3#+=Eg>bl_LIOlR#MJ+JU_Aa&GFLAz@R)p7m$sVQP#5N;(3J9M$A$jlc>0 zz$SZ;^VykY5~&XOGJ8{ec352(O<`k5*f5WzdJa^gaLo4z=Y=r^9w#-6KUb#_@1D0{ zaF24dRsHJDzfnQr&yIKi>IL>wdYIo~C))F^w(-r55z6&h=ftO~UAdDi<9Tkx*=#~l zoBQflx->EB*UWE}V5PD>WpS)~A&$32mWTFA!cL>aVQ@hf?*RiI?vF}B+o1eLuvJX_ znkgt|{wyR!YK4tu7V3y_Df_Cn_wFZds@B9E&Bcc`!u3+e$J#cei|FG>3mf2ikE4ZJ zDMb>MelQ3wSr{z#K+k7q?LL)fo!|y2RH{L{uRIkOJ3~)MJ9>H~`$xgYuSc=mShioD ze>dgJRt?$=A^Gho$&fYV!+g z8g=G!SiZh|aEK!hqUUh)t4zgG=8;HYBG!I+F1Lx0f4oeVRQ(MGi&wFKn?^7ilJtti zodC6@s^-IK6lVB+M+jab!TF?^bb>jYVliK**O%gY9*a!Fd3CU(N6SMK@=BLT3+wGcVstym{okd)VrR0 z`QYqNt<$T$OszuWdcZyZvN>>_?&p3cbWeDgoUl4z-MA;)@qRMby$i+J$V+vcK(dzC z(57HsnElhs8O@C@V(Kk(G+s5 z<2{BAgH_z+p#^=o7#~3l((9zMf<^ z;83;U;j}cZL7Y)ZsCtTYCPJ><$>jEmmhMZL9_K<9h}N2(hbUqrlaeZ&-C4L!`a!KK zvh2%Jy0IcWD+HI#{VZc0P{L(axHVGLWo=y!i0hL%H~R;qYuF!9k|~A*Aufl5?;OdP zW}A}Z_|daJu21b-=ftQ_lNFkOq|Y6)k5QE)g%?QnXA|t^z!3~g?^=6+=Gfm6Sv8Q( zXwfN_V9;I9B)CWi#is-7UOSx0$ttwQO4WyaU#;~K+of$o-^N)Rr!;UD-x_yzw_iqI zd*(_=nqufUm8d&Ngp2MH;~$x8Itlz-({8zq8sV*+697}8lzbHe1%plI(rrboLXuy)XUt&L=$c@oR-?+>Q#aTRtNs9oztv#fNUG6Vx7X9-s0Kn);8!wMT{ z-ZEQ!6TH-au~9SJ$-;sQ=LU?CzU^fcWu9K{8>O+rB09i>_NB~-`v3IJ7z18wBgMnZ zOXlJAc1%j1U(->e`KzX+Nli-AZK0p^5QEk4oNP3m;a%+XhtZbgxC4%>2U}y2x(SqF zbdg!r>vmRo!96@r)7jKlA40v#;F!4H45h{h!mq6*^{n)t0j*JR3#2)i$d&I+(hbo?0oa`n(o$$5m+w2r64^R&Ey9b zb_|3sS+m$0K!uqggzGS(9_#v*>zRxf)3`mYt4OLgiNsZN6eCVbC&B zL-V{W@ns{6TZ^d9XR{Tq>#s3mxu!CX>ME+v=36S2FSD^~7lIW}1NBXrmXl5wPiGlX zfWS-?hZuyh^Q2>><5Rb9O-fzW!70eA|L2h15pNu(so~aaCrA-^>eIuQw1FQ^i_}L7 zb>uMtDXZNa+6n^#rlVxSm*gA=56^t7yG<_SU#`X|J{>9-H3aOIaQHj;7crW3&iB7( z-)7ZKCekgnunehujv+lRL@sCjan<_NDPaf;2@_?RugdFTlOrkW9bW!OJL0L=WR(&o zio*zUwVjIFIH^}hydaQEO*h%DQ~P4$d9N-e8?0?%;-L@H58xXAx|lX4P&_E z!RWl6d8%u3w1A4tOTiZSU7)+8gS)3IiAmqHS`y!WhMIjA#)O<@XMKM&j2*6&u&Uf~ zkO6I03?_r#s5oV&U?qs(x#z08t8SZy_WSWh*-n2Kx=7orlzQrQ!@#qTxSLYTX95Q{ za~k9TS|YJ8-kPUw{xKZSUVzwMM5I~%*tw6?4~i?S_#^%7OR=*U)@y`6;}4a_sfjr> z8R9d{+f@+ez2=ps@6nU7J~krr)ZRt1{K&7 zB(%5qY2MuqNMY2wA=LRIV|6!XNed9v;@r+Fxx4xam9_afASH4vgMx0>5^C#ErIu0Ngf5w(#gSa=r=#B!%O~D{6?;%J;j%&gfe_ZT4g`*!o8nZ_| zN9ReIe%dJEUyhL}*<6l(%r`Hb;I5|9VJ0<6%c3HwPZWgQg|1E3+hHcy$(%jFKe zd~GXM#PJGJ(X&rjjQ0=+KYF;u0rU?Fprbh^D*SuPS%?cY(n{*x;C21xaV7km;R8pt zKIL?n&o7eo4f#WPyY81CxH~&&?0Y~J z;%{EYraOu7uV9b}&f=yDTlIeg4NQNO-E!ss3?Bee`6|J0w~@yR;5fxziZqe$a-Z2$ zdP=Mj&hvY#cv%m$$(O`P&$)NFmCR!z7ullON155f{Nv`^PaS`d(SWDbv@VtIcAYC7 z?cLrx^0F$6*xe4nb`Bj`En;hE|CLaFl7O}XYMbDYykh&kMOPnRb+vtZK6ytZ#;}1g zP1k8OlbF`6fyUd_PGuqLXybv3Z9^O)qA25C8E8B5q5?ssQtQTcKjjAS=)JL8rWrWjo}osybrThi#dt@N=CF5lB^r9S;*c{J@Q*PPr&&V&cD=5xH7z6dmop)GSjx&H{ zB)K)ek*d{XngSYqI_CW5WuZ}psNxH5@e$Ij;(u^+7#o-8LOf z*G9c%>@(PAKgh2rIXVIMcXNiCYPu=9l%+YsnSfYMX(8rQWe+}l8Bfwq`YJA24xcD> z_A&s9-72I%pCmaYVz?60(@=?#%a4|5Bs}!O1hT3a7 z#62js`;KEmAC#|LT^|NY(hmm;m62`u*|4!Gc$vruQ{-m&2dqd_qf=5stP0uP>ejxy z+#d^@th&CtKFr_PEuc1%ohecE_dIp;n!k)&2Q(V=r4}&|d%rj>1i!ptUH$^!Bhh~X zqNMFDK<3nkPpBy+yq{NEN+%9~A!vhdJf*?-6p*SWht?GRV#8{+@foUPj9E(khtr`gy)96tl90L9GM0s~d2veeE*iocFV!Man;!;74@t;G& z1P`-ptsEObt6d}1Pg*qkmz#MhzL7=Sk8)>rKFmSRXol8_jWwoTR5*D$N5nM4b&xshfYbC?W z(5}Bh`~N)L9}qn!7eqZe%es^kh1Miuj;=b_?&P*ZWmlH%_=7=;sydY?o_>S^SNso| zdHXXTf8X0rs=~41yCU*Wfo(7Mu=swry(+FX-^CvM>oqs|TSCfWOJiNFoemZXkj0Z5 zJU$3iCIX&$zZMfX)gh(|gBXXnQ@6;(wbf2p>6Q3sfiIS zT$0KJcoUh|!bAA>8kJp_G^3ruDKY>FuQG-T^M6#+4mNt*qV2Y=<_-7Hzn)%INXVeB@w*&#A z*C2|taVFb!JEAjS3H>~Labhu9yiXhYArjTRl2Ie*wobW=&auPha=jduzJ(}zWMXISzA zBI2-PNtKgKW8MQAlT>DLYU^-dRs>v}?FEkmFQHL2gJPn;ki{pef6dN>mCk_9{&$i7 z&yw8YP49zn1nubR0*&fz^$XBm`OOvq?%@NDzK)l#v@?oP=16vUV^=%Z%7s52Zot;) z>>BGvUnDnB%O)ZK@SW%3a3T~9Fqn3T@T3x!RIJHRcJSs%*wiihDUkH+SsKDHcu$WW ztkJUx)ulE#WBn+$kaHB0ZwnQd<4SRHN_N0DHdSrZqeXIZ~)!S`2l& zZuPZ2AWGkcM$U8hti=jip=`O&4omHrak0MvXmNY0LMSThxa>zoX9{OWAHHjS^J7>+ z5EK&QIp64UYGPOa@@KPoFO)L^pf~tM=xc#A;;$W;1kK&?53M7V}pv3hi7%W%2 zEr1>S4Gt1pjNb;zqin{tft-I2kN3Y1iv79u%Kp3Tkg+9@@Qe+h@Amn6RB87-S}Q9v zYiS``?GMKRDPc?>q~cttobLPhF5$((pE-h405mY$xFmG$oDT5MfGbBqK>&bFa96mH zy?Tc)ncWBt#u0>A$KE%cIX^*x`dAM1BOGF`yXLJ?$JS~Z9ZJ+wSS7xqVu!W8tOPu+ z!%9|o!7|$#cpEc5tsOceKR;ZBUt+LWeig!N)hdb=vs}-l4#i+0{ElmHvdZ71B0D3w zFpTV0ce1vu>1}Z_rDS-mPu>AzbGjhZWm+u&^xV8oJ`UELuvqY>{?)0tY*s%3Cb4}G zS8}9<;OUf2a+TuhH~xA~d&2G%Ctz^$*`Kp(Q}M@qBKlaMW5J)N&a-B~uftj0iZv9X z8F1iri@CLM^R!`jQlKd8DaS}1e?Q&|iI(nk=p5j&F2 z4S#1%*lj!WB?$hH)ODHVwk!CEB(iTp_N@Afi1NNoj$=BFxv??9E{Ib&5fSV92EDW~ z$*Q)D+0zFIO-s|g$Bk=%PZA#Uu5D3Yw}bMJ>1ZMNX882w3p)CyPq`0% z1Z5fbO7JKV9u5v%L_`?x%yDl$Rq=V>L}*vDcCSQ+`m%Dp>n>FyouP-4HA?$*H=oOz zt*3W+c@l$mD$+R;j_HdYprfE(Jry$zs8<+UX z&m+Rpa;j*@*I->*BI~5xJeK;fkRxd8O)m^c`4P^sdI{RQ#}Terh6%mfb1A{# zk&*qFYX*!V9HpcxHlX_7b@dQtHcJ>rU)#WAeUq@*)Pn;s#h+A>ud-z)Dc#)nTwU6h zYzQ&rj_p%)Md+6`hwt$Uv@Gu&;$nmDj2ETWA=X2Mk=bZbuB-WP@z*JF&RXL~5vg)8 zs;>G=ahaGtDwIBKdOEaG$xXbcH+Cp$!en)UuQU4?G-``Ho~0sV%kpN5MdAW_x7>Qx zlG}_AJQ*bJdoX8b()95el1V$jntW4UvZ-}z653av2%Cf(;T;~tMRI?>O4Ixwvp^^ zla<`nJ~=AZD!_58a&sfhl&q4sf-o1tfc%tew7L@IeUHHCT5Tbbts1)P8^f!40e?QvW_!Nz7v+ZuT{ zL*q=QlD135nu;ny0F9n#lAzcgfvV>IQ%c<`!i=-j+{Lcl-2B4E%d>@}n&U-_i=6U- zE5kNoGP|>S;)zA!7^WPxnce531Q5hBXm>hW;*Cri)~Wu`TS z>K$xzpL;{q_^qtSawRbFHc(acT9E+};G0WhdANd)y2;9$gW@OQHKt;%;M7o>@89{- z7#r8!wTU1_*q!Se8aU(U1>qPNW(-z};bAb@Lm#e<25VpN#IYJyYK_W-y=YXyonmA0 z?p$-}qY)||TGgM}+!Sm0L(v)ava3PsiL`cyQ=8z#e3MFJbbI_nv$T>4mO0%Y9OixF z-zhDPW4~UdYneh@*vxQ0d0ox`VFS0Jm@jxF2j%?A0;K@YC^4BS_8XCODdxk6&h`BQ zgq>uOIqjdXwG(0r`&o`l4UgH?CshaRKK^_rL4ILc7K;A1+PznvB_%m< zKgHm_AW?BgMR&BUflY|XqX|$|68wlIJ^uVXBH->KhUE*9k3c2(6`WAuNNO7?B_sfp z6D)hgSiX@6()ZxP)0SMGW#>VQ;uP*LQEeU{*fqFrdpCil=wJ(a_592?dx$?#VG?iRD`LK6M~}eSdLViKdk-nw1slB*1%2W^(@0b;g$r_9C|OS zcH%qZ_s+O9cOuIBV7@hzZP=$FW0HMU5RMW>iX}0GEaM?}~MU&o@`1dHP*rd7D$0lQM)hjVeP_oeNk2pD9>u z{vc4j0zp(uX_8N?!G=8Dao>_nRY5lL@Oa+30T1k}y6}&Y!g8maKQ%atm7qO7BR=;bUmfQACja$Yk4>|$kj)}7(KB#hPE`=m|v}$3S+2qMlu!4EV5ZsKs45L9wxzBZza`9P+mpz#)@xFjuT6Ag!b*IbWzNqSij`4gw|0 z#XF>o`2im{N)(lY*{~J)SSp>&iw7+n{~H)4JqX|Nb9JI%AdhdQ8ady!ct$dEhIZdG zIE8u*06nsfJzmFJP;UD?!iND8lEU1dxJ%tevgh(?ljtXaXk}v&mOT7UVB6A$!fXjf z_BI2uJGW4ysIu`Ga9K73^GT1akE0cLJvZEL2YlyhO8SkbziUh|yCWfgST(laatEy` zyifPZYMHrq&VaR8rheBQ9GD?LeF2zE#LQBwpLZ5q&ThHOPxio!f9)g&mdYONdu!vE z@|ehYs+HnveYHk)h&PAMyd#C#%S)F(*U}&7l0MON?t5CCxCP}J7mLFgHq!iEfIvVzPn>mZqST_J5Ba&-=&brv3GJS z;~D(4wXwz71e+baoL>B10PdaV5n+2Kd+)dP4S7ig8wpb)lsX)X9DeXyP8@r1#RwuP z(UYN2C9Zd%obNy@h0|4LiRjE&FX0joi@1jtyQ24T?W@QUV9V{0$ z*fo~vm1PUUuZ9lRtv-`3D-#DDRV*oxPHc=OMu97l%h4*#$-<+DiB#oI>o#ktXxjFI zEFI80@72XL=?*o|I@Q6Q`_gOdz@RThpuAT4vOl^MJT)(rbLo<9(dTOkJ=n6trTX{RzS^+msoq1x_NG;Jy}ti}C;SfeeA7hve5 zSfhn@zrk)fgj0gN@>^Tqa5>iXst|5}ZYZ)>U#d(k=waZ(0`xsxA0J3J6iUX~9(aRS zpTlrco2n{I?kC{qAfEYKij{Q}U8DmY-}WFuaNdF7g!6#TiN#j+Sk=zLchd8JIpQ9K zz{BEjBdhAQx-p}|7&M!S>m;S6RaX8@YPlbsQ)v( z&>>uATf{u^5T?CcAhJI}Xz(zVVHGmNAV2Xj{g{X+JuU; zWoWnt3OeN1{@sZ`Z_~Tryb5xs%AgqOEe`+s;7I&2{^F1WdEA$Eg!46cSN2_0NW^}8 zVMvk2YGJF3@63Ddv5mE}iYJ!NBR+#PM{bIh;&yeeSk#_X@F{IBDa~)HEgKUlg@q~~ zyTG6D)T2rSQI*pPi=xvg)iwNP z0|dm|d;9a_NdY&;J2lkt&asA^IQZw~!C@gqZNc&NQjRgLvs6nWb>*~`Gs?n_s9PLS zsAI~Hbx?hR%Gz3@UhW%O%U`s5)#Vd1kaYo}0pH^*A~$!am1>mq1~J)+;}}Q^FRgEv zH_X1)Vb|eNS+OO{L3{*;vkS2l=D;aG!$8+EU~yqNR^N~o2%a4>d%OLl^$i~gid^t= z!j+2pK^yvY7<#c{g4eW>Db796KVlXL+dn?@yzfvP$-*&SOM~nDY~;2@tNZa@R7-Zz z^zM^47YK<6rMS5>k=7M0v@5~}Q+bGWf2q`V$iyw$pF#~b;$5w2W}8f=*QPZeD~$Wu zov?`8<6{NVM3Fv?xUSC>2}Md(B%_ufpP$@dvx$lEEwQ3p|DgFL5Rh4I^(@}#a!nM1 zJ+Dz3re+4UFfY}QU?xq#x`oYIR%;~ql1C*h!u@=mKl{rXh>RilpR+d!Rc&zz=K-x7#EQe;j_l#U&oxo*jH1QjA(5gB*b zlO{NW?0p2Gj(bO3yB6Foyqeq zxLO-Lm9c6vX1@8XymH)3)O)UKlWTd?L(9pX$s|{1$&>@+oW35*Gk)m(f|j!6r&_bT zA=SazR`#5H>&2c6dBb)2!H(r>^H?F8u?k79oFM-S1>VN4(&c6ay_{`y@5fd$-)6wc z3Ah)4gojjy9YfT_^R~wr)8KqyxLUict(o72)fX-lC%gsphsd1NYn07}tjBLp5hsVS zO2SpCh|JPxg05FBUsXf5lrWOu6Pa|mA1OkgGw>tgNR<>H(}9PIM|n>^AaHXJ8Qv5F zMK5}j6WL#ims=9@B-Psdy#!gC{?FC$=fAPMI2tA{nM!Vvs?APyBi3JM@@htGl~Hl= zV3wCY7WFGDPKA73kWt>SOzZ8q2;p#OYW$Hy&)axXMSkQh5Fsn5gG#G|*?Vaj2_2f0 zr||=2rL#8zZ@a#wx^}Y7d@u2l7ZM7WP^NTZXLprRs=eEIYk!@8bLF^9b`uTp|B-!Z7AFCWqiA1MojMu8}rLztgmTuoU%_+{6?9^yKW~}B`SEt zX(Lj6Cj(JQO;nLYP1#5I2P+>!dLvT+?i(}QVtI1}isSHz*2iIw*#2>n`pzR!0!Od{{eGCEqHv2xw+1zl@3KXTIJp z>EMyD{Gbrz+xnO8XMr%h{e>F1Mk*hc+WH`%<3oOXK_QeQ;w+7ZZGJF*EJr4@kmTaR z+Fz%dpt^vT`|*F~_nl`kd^oo{r(`@vPN8{!amY<`T|rnJWl|RoFpzoiwXCqb@xU14 zNsQU~?gU5{7AVv??k(l67-45;>CGP70#!8ZiyJDT?SY4Ere727QyOLE?At=RC*W=g zo4HpY9KO6ep=)tWX2R5GGJ-n)_2cL?$@foZl%;%s6V?w_6YOUA(LAMf{6O-}CnodL%bU`;FlMy6a1u6qk*YIcft`Cm!% z&+N(R1hEwqBHw3E|LVL?J5rvv&*hN*5gq4oBj``esG~k1#$j z-g@XB+)Zx8Ihikg^c+I;2#<5L+k^P8g8H_Se&T^DIs}FZF+@zqztAck$vW%@IsQzz zxfBV}SZEms0i~5_FTYB?KOaN`JKu}hEB2vuGy~xTJ}oat9Rss4kNkMBH)g> zS9Ikl|1)S$QTT5=;!PJ}wl-ID?xa0S)w*EA8ufI1u3B1nt~A;(Y%W(M-7Rd7FcTU{ zwK@5tBaqh^Rjn_fH%Dyo`|D}^0)sE;FALlNclmRuwtrGaW2ZpaB(}TRw;w%&`JbDR z#lP#j#}#B$4T!NBZbcE>oSo4G209g%&ZMGi<+KDK!Z||I7adeLEd^GQu%`-$_N+EEI1CodT9nA82o#?gA@J;nhGJ~r9T=;O2(Xu#TJmeHwP*#DvAfs z{I;oT?*jK<0@cyh7Q3;#FDTdMBGXAryV$Nm;BWyWqz8$Jkl2gcE+V0#3T?DI;XEM> zl+ohObZ8NYUPeGbFgUzfw%ODpp*LOP)F(k{n-dxzLz=I;wx}n zl;7Ih<0KYO$>{5q$mUyZ@qe7EwY9z=JuW$k-B_B#Xl`|=sNLGqQGPfSE>voPtk6|O zOPi3b;+Ioy)BOJb+I!2OxVEirG>{N12@ou}1lQm$0UCk@g1aQRyL*DWI|O%!#w7%I zcXyY@{VtNd-|T(vd8_V^TlM|9}vU(hCgz7JARLwFIP5CW9eNHAU~U+Sb4 zeEM1^SD?qT?^JLtpAM94h7wCIGF)$vWErZ{ZSGr1YbOAjP=v8ICPx)f$hP_@!vm@n z?Z=g?ql$F~bTjS_0^dmZgcUTq?VIQ<_^0_uB{!a)q|3oq!8-a z3*-;T0?u+axbtpD-%JQp!|M~7)~m-|3cr5We=>EFZ!2(wJ=BbOMZhPT^AUEam0)wE z_5{-(l&CaCjg6tmw`XeInvtRTlLg?u}M&H0b6puu7PTIO0s&iz;yk}o&4iy zPGrvL?9au_QT>78g*)G^&O<#}{uUUO+g@|#wSFixIvOYu_an-?s(ZGTK@1-;BkR3; zdx9#Hg{YobYQf%b$c|-ZwE5}LyR0mKaQ$8Q^Uf)jRET2IJqrOD1xALN=?j%2vt43yih%Nef}gB@%?0 zvL(~81uWLiAlS2DLptv^08cJ$aU_X7)y#I>RP1NTd$T6U*ulWFTWi4(#+%3)$$PDp z>GE7QyFZE3wO)}nDZK%Dau z8r(`;QX1KKTWif1Az;QsNDuVv=7*!A=zu@nTJXOgF5~pidgJA@nxAUR&1iLO`T09pp_FwLe^l+{b3iml9mq zqKB!p!*d=D#z~W#nU!4_eTbgHy*3qr8xEu0d4f)tIeg zt|_gsKA5Vh^m9FS6Ff-Ec;$7Gv+(xNvWA9r zq-NjsDUl2rD5z6QL^w7x%2{(Z2fyyhtae?A{D$B5gx~u@fpiO)lQzyLqGfSyOEXaXT-{@n)*C>xdTsD*Mws{fmJ$+QPo1 zWagFp5k8*Iv5gN>?zT8~=)9JY%EA?T6FoRgHE?<&vo;MW0z>#kY;-Hb(UhsZzXjg; z!(pc1c#i*_BEb3F{yeuAExbnjd0aPB$NNi5E#e4*6{IaRQ+}!087cEpqVxEr{r-Y@ zBzeeaz&twj&RPvBJyS*~XP{g-aj{&phUq?H z`45;ZnXAE+JFn%=HnQat%zJ@ReYZ$4<2fKB`WTsbHV6_@T*eIO&CMh1{*Lfm;Ly-5 zX$eH$cQKu}1+{0d#(60yB-Gdvgg7d?H5raK-30a(uV5g`0SsS^VzBw>l6FR>QcmA! zi4>9{`eeE(+s^R}4r2_qYTt1UmcL@$3@&$US=ozmYh2Ec0YBCYk!$VtZWYPqw`Poy z%K{H$D;OUfeFbl);dc4#qIDHlMP>DdP{WmDzN|riiZ?n{hAxKwEgZjWT^=PPe;!{-#>+jwh#}2@gAg2OlVll zlr)B*a4!ZqLuq4(U#Bu)??tU%uRRDHk-`$#tqO!yYi0v`VGP+J=@MZz1I-+E?%R-2Bt{!q`;)}?qh&? z{$*lN)h=7NRw?&$_egD;uPCsB>p|kTFXc`Yu>;*NnX{LD&cCOE0$}?16zR|(@=^Bh z^$A{LA0;pwZ*FewuZwm*aDi|Umz5z*ST-@6(q1FwPjaZEgkr1w78w~$Sl(6pWduvW*2_oNew)11XcP*iUNq7*{HZI_ z!8qex@-X1%wq(LtKnE4D9ex6(z9y?GE~M!&w#7_7h`lKpu4T1BaU86<$xl1?*e!KzVuGhcpq*_|Hv5kWYGNh+2y(R@-u!CyU5 z&n;Gkh<5Y`Y?asbwWtD15(sLOJ1}I0K)zknkG*qsPyT>ZnV)#_ATs*`qc_ecT!n??7V zJ7|@=o6jm_O?cDfG}BH2S_OM4y=FywIVC!*pCk7eb;U+gckk1aTW@1w@~d_qFv_q} zhP1-VRg=B9?k*N04C^_(9qiCB_8Wu}_UZNdP=2T+XSHPMh~^VQ=ex1Hl`}``rA0bY zmc5x`46R^!NpM|<)<4w{U%`1yfUQr<81utVa!mO$iTwV(xP-*_5WE@ed7m;G85H8= z8z68dAz$Z!%-y!$EZh+EJ}Cr1GK3gBf&NiPG(#abYt}|1HBHQS_ou5!mDU!z>4Dl-uYxTrskQXOlIwWLv$`a-6|NgjOGdFLk((>8 zmw4fHyn)OFdL#VmLhJ)70lHa!67O7ZMbu95vaH*%db1vuI#LtSsIqkdGRkRX8FD~V z9EW0u4BU$s>G?oDLKMl~aHDTJBEa7x&(=sWC#%;jRsjEE)!`{YkXxeE_N_O6fIrH{ z@k&3e{u3GK;`|C^-Sc*_k6KHZhpCz`2A?P86exbREqtwPvBtbQ%T`1L>ni~99rd@K zwqHVixiuNM@McP+<1M@EbyDFbZ=UWu?6U)lm8oY^>FGi}$4q$$jveJ6y8Y#vF9|}+ zT4_?6LE6+^)!%Bml#{2<*lB2za$gPa#mjbee`0$}&}qLE=Pu@CHy+0RSMJ1U;I9`9~rOUo|N(jHJc#`EOjkWCyriAy<6(h^-c7+E1 z5k0@lWe>`jyoz>RPDmZ`VZBHdA1EP2{kR`6BwH)vxXn6SCn$BjcM?AWqrkAdIb}k# z0@QXzjI++WYZzo+tOoI}^2VpzF0Ly*7gNA{-U1vD(sZA~X-~kP2;Tb+@cYH-c4&mA zVZ?Y!>?Cwi)4dKW&Tu;;6L~J2jew5V6HY)Iqr+KQlJnB6^JGobgFHmtB#vVJ$;Cpw zcPo}PZ+eMIF>(xBn#VT>jbF+Qzg`~bGwiOttmc-dEy;d?Ho9yrL;~Pa)w7>ibZ z1iF4%J8MY{W9j;Qs^lPDhn1U##Y(SKe0GO#w;-_coy>ckYk9_jY*t@4zAuvvu+_`T6Qwh?ilW9Q)P3L_GmrO?(sU0Qx?>YCYQ^{{10!V&le zJTQ>J4!%txm(WRQcDl=s>rLsoaxgrkT%}swCYTpT%UH|Q;b4O;-mKEeRI76dzMrc} zHX6^Bz?)?q{Gd3k(EJeM{wm_ZzhbON);M3K*2Smu#&-PqaT8;b@QZEn_JFzw$O{L} z4(JnohY0Qw&qXOT`t%$7yk?eHShCV>@?b~&0MG-k@4#~NbmanY25WB@OLMCG>$52K zOi7jXfxFIar_iX$1#w+6V^b9xO9CK(xPT)xF0woq6IRLczE-kEt*A5Uu8+EbOs&Cc8qN1jsbO= z-!V>%u+{*Q_}0*Hq>R#y?GHuH*$1lwDb9;x-)|#BpHRI!y-qf%KIXf#_=Yx7tcI9Y-61hi zo71#pOG-x4Q%fC9FCOR~SKO4mah2?RVR*v_+ZJ01!$`WeeS!3%j;zyIsYE#YTf>)! zP|yj&zjh+XR~MZ8z^_lJ zrsKzNBFCsKpvHF2&4{r`C)hId2Jj~L;4nRWM{^@*ypZR7|8m1(;=qwP!@wh$BjmvCxZ;hYkU<6E-g?~Q4@ zCqhJ9Im6?(V#m&lQ#7$sE6nnX#$;x>R1<-|RHDT@yw}EEdB)(&+w;z>5ooiuqxUvU zG)eahb%|nV)MDxyrnC{GFeCX@iCxtuhLn9*5B5HGfz9s~!;woQd&wZ?--@=H9t^Si z6Ryi*1S4`+TipfKo}ca0J>Xx?4VNKX%COS~khUkhRNb0#&&f0G&6fL7{i5i`nTu8l z!qtBTk*#H&e$&h-=>g)F?N>?Mmc-ib0#o*Apr8)Hk;r}J$0r3cEGb`I-L5FUfH*Hs zRW{;REA$Z=F;b)MsGOyXPpw!zyh2K)+kmheoXpNX}4^5ZwZP=*A;+50LY=& z6J~eL_A$1%)U-tOORc&ItT!Udpfvc5R)NVrN`f7wJ{~mUzOe9AnWgAm9s17?A@~d* z>hF6My|U!BkAkH+^zS&w4R**gsuX-@yx*{Eha30w9eRJaR#-Sgb55ia`UpZ(uQ6ev z)vWroU_4VAQhCq#E(n}I>tXK?Vp(|T;bZG|0^9ZJM@w;itmv>coXI(i3=U?=s=9y? zfB24G@GvsquBb%8_*GtfBSMi9NzC++?Wfsj6xL$zIrkIm#OrMeBe{}nT>5IaGI

n&*IK0}Y4`0}|D0wk+;`6Y2BH zNiuJQW8!s;?H+9*0aIgQ;A!=44q`4mZGe(%p=%UQJE8R zr@{#phzhy%tcc5kQ;E5R)!M$PZqpCp`KyY>Z-<(zk(FL*a2>mVKPRH%Wja-!n13Z7 zmYz^nt9NIPB7Y{%C5&7D(`>$0s&2Mi+;AV;O|Du&emNn|^}ttiK%EfOPz`&L<=d)a ze0E*b`q>zZ_RiAB?&{_hUhGNv_bvsm2DfIh^-R!GS7`Pp8;KW<7Zdz-O6 zFL&uXjBjwbJ_$A8C(nNPv{?_+yPaGnyImsBH_|`nRNWbavd zd!~s0iVRIM<0|Y7_*@@2FH@g@KBJ!&#XLOF8J=zb$OJtnN}7nj{7L5uJ|>u0=f|b+ z7HyV4_xbL;x5C{On|!hK#~h8?ORZMS5v$SmY5%5#^D-IQP?PA+;|r>QW^+1K`vVk# zWh}>qT0!bd|K-SnnxER}(*rh|g6}^1>FvKeLMP03kSXz`wEnjP#O;f2N%q9F6ir3!5E{;@m05r*ZR#;CHA(RNVg!;@N>>iD8pVd^fb;b(+i<_c zwu8VJI`$G6$5R0%YFPAFKLr-FA7mMuRE%W#(BWIq#JR!XF9jSJb5o4(-ciH>d_Tha zW~NGlmo7z!mN#a1B0X0!II)M4H#U{HfwUV_^A2;lX^cJ^l@`nD7ZRc_0N_;#?E-qT>RRtOK5!G(DEJ zwD-kII;$)0D?w;!)3o)OBJh+mmV5QLSEgCiG}W}9_PmceT@%>yq@XtGhs_IEFe24* z=uD0~%_af}bDKI>2q%5{q_P#83PUB}ss-D=tkr~3WD64RDX`N%n7(vn;!r^07vq-*i_rO_V*!p6&W+ErAJeLGyo zrj(+oqK3(^PF(R3k7AIuUkkn9q}v(@ejoEe;kDgv@ooK5^GiBcXS>V`AEcp<=*Vb}+k$j+Wkbq~-T=PsvwI#2HMpFD zN%L&GB8}lTMYM?&kz1*e22it%FDvc0DJLG%QL$bPS5a#X?leey)1wCrrRRo+ck2juV@ zODVM3PvNM6JbV|BH>F$yfjR@t8zFW<4TZPzM09EjTWR!v?deL-p}^8#CI@w?*}7iZ z8=Q#cLqBJ`K58Vg;ann~M=#FJi*E0oe2pmf74|f&_`3uSA9&t3bIY0GC_&UFPF z4Z1C=1P;{D*;Kn+Cl=ZqK5P(0j4jT5@Xy3l%a^00|BMfg*c>mAOHWbfg7Y-RbY(Ix zL%RbuMtW4uuIKrhG(jQQ$Ituk7X^-Q<=93nd&-XSxm{?OmaH(;Fp6N$_p|R3#X6A< zBH*EcNDDbAaO4UxHut;V>{PL-`0+!dc%AUdQ~;O#VOn6US8iNs$Noar+xGU(;kuWJ zc|hf*eOH614j)&*HxxGq>dr|N%AvEE=dnkdAxIIZo?@k1ZkJGl3_QhF5Q9-qH+JtR zQZLPaE~;+*`06WHuV-sCBXk1OUbRaEB?m($3csqR;=JvW%cjL9d?_g-u?gpV5BZeD&&_*>&WN zlX0KU*NbSeftH^1d7wA*_Bd<*8tj**4VeNb-7Mw05So0++Z9z)RV1RevfQ@y`#X_V zgx#n(k=22G)6?5;FohQ#u0*s*=obq{@>G7t`pTUL`QGZd-5oPN7`G7L!r7x)?mKOp zVvZ}puXQZnqL&R5j{5~rjEf3`{5w}RN)VeOET1UCFB=b+vjCOv8f4^AOadwW2st{m z7bs=Uut$Zo>-Z!1Axo@wFq}MwhI_ z(`bdSzjotu4k#sap^3EEs&9rY%)<8$!y*y*^KgoXODYRbhM4-fN3~9yy5BE*gR<^j zv(z~n?M#+0n~>2~g}^Ia_O(WCijT`I4v}sA!fZj%YK47z1kgltkY~Y5ZK_Ie+vW-4<9eiYVc!5JPz~zbD z{8d^VFBmFcFUU2)A~2fGu65Y}n!-tjd<+>Y)xb;OvB#fABx(){WozVJQ%_(rrW!An zPj%B*jGv6WLo~PP5G16~y4B<08^D&iih-bKZ3PDlkeTe}|8>#Px zMyp?=;!8qKkMi^FINrR(Cr_}jvl`lNKZsOgA2p6B?a)REZ7-lC2#r$E4SD6gbyGry zLcXrG`yS&MhQbAxrBUfA9YdR_!rcr?YdA!6 zvdY}Ebf0sg&mmzaTQoym-YCyFi$-U->W}Ge7v-EP-0hb;)HFNb;ko5NkqU0&T8|Qy5A4GLn!d9o(wZL4+%mOV&$W z#2Tp!^9EB(o;FKM;WmR)jm9DgT+Z0hRNF)ktmg-OLbod8d0)6@=sUEJ4}cb@xsnBX ztwr2PkJZ*9l?LL7tkSb$9Mgq5L9R)lp{gPOqQ*$Tp`i_P8DdFf>B`+nXx0c-NogP= zP&~ySFOrpQikX^HN}3OjLH`I$N)T!nCQhnBOzi+>$wE;@fwrs4t{P5#%?)AC6zv!e zp>&`8MR1WPSuV7v3DNJefw*3;(wpB|WI|Kucvt%9^RDShVj_l50gkxj#RlQ6(`-j4 z;SRSzEf))xuIh*K5w(Fn`MlQI>~SN2iJ`REUKHO=*bIH`&#Kl2>j+wty*{dAKNfi3 zcxl&jSBM|HpNwi}f`;WseI1iVho>U8TZcqGFaj62>o(ExV(m(HmWzCQiHDtIW6!zU zTCugOe_WZJb|bizVC;a$K|;PK>kgl?q%jtvG7s7$(l}@q-kcc%tu+b>jx+Ckg&#{( zH+HIWHC`rmj$e2G7&_s6wR7EBnHaxhd}`rN2-*h#AN!$xw%LE6EA#8ry}kKqBg9=SZhA=5v{C8T zX-pNy*KDEM&xOz4fVqGwc7QVFe)R5aXE>f0_k@ncOir_|6JpfBE&hp=sKJ%}o{16} z@=&pd-J4~$pKyA9#FllLqazx_mh!M7EvyPY$kda>8IJH?43{!A=X5^D1v=v;KXh3@ z@0_#$0&lVaDQqvFi0BV5H*{02mqv1kDXfG62E{L|m2iEQf%`wuQCNN~pr`+W&MM(T zeVi!MS$Lsx+HBZH*Rt53@IiHwCY_ZQJ01qno{c;pASBv<-)0}D@AK6R2r|Rb_wvyD z;#)+;v#}MYP_yK%%`EC%^_&G8@i!j1h^m($#NAN^S{4QYj_~K5ZHZJXi$G~qR0&c< zn!34?Qb$m~VKvL<6Ou{C^Snp3NR=grJY34)y3d?qtB1wbI_YtC0EFJu(#{g06N%Si z@ocH69wX$)B1gUc0%3HAFkgBhcBT@;Q?xyhPPN@~FUWe6{u63=KNT5k3^C|=m2W8i z@FS{9p>ow+zGe=YZ1Q1`gYXM^B?kS`ZNH=wqCVeBkjP*}YPUOX&rM7g&{bXx&FqcZ zy;6lno17!j^9j^4w0!w9rZ`IfSzq5p6exK-9SNU*z?ha*DgG+*|3;P(!UKk*cgh!c zaA+<-@CWqT`m)&a1u>;egVe-{0%PL7UnuU#>(Gl38>So>Q>qK zu7j5Hxt{oxy~vB=qUYPO3TJYat4>a=2AeLNJKbx`ki?-9DXDG9lKU+Q?fp6%He}-oQOtbyFoBDh zd(ner_G^Mi@DB}inu@CGaA|etQ2u&g?gtdb@j@S|E12X}>IQ~VdgVYi*~8$gxQUW` zc|TuFf7yr=K8p50^tU1?vEH&V^lG;S$j4MT{ZVp2`)urs?oAm79i~H=lO~M!ZpYaz z?_9^c{sGvGqlUt0a%r6Fv%ONX$ABw+lAv)sQS6b-s*&s@Cg?9cWg6o?=FR7>{;Eza zsijr=Q=o&-fpv0pbaaScr|tELg}RaVMG(Nm(Ncay@~2!J+4Fu+V96JIA5su|Ul#$u zhUm&)qGxed6^LtxdUMm!%pCL3-JBD|9KXKA*C#NVEUSKD(v#OZEfxyHN_Sh-m&df} zM|(0t1$4D8cg=iPveA+cl)^;A#{*=Ot&z3C`OaUYHS-m7Vr~`um`Ej4uk(Vg(sjf> z6Pcg)Q4%ZoXCx&^)<%LbSkeAjyLr3@vO*^Ng%rGGuBn!D^MSoI;Q09q*o?&ilSz~P zPG?~tQYfW?Mux}@-sl;`q3tW-$|5AHRe*3iD|Scn8fA*m+oBLi4GzqESEPF`Whnl1auB6>VblKahc{LE%-YSESwNJXI6vCfBL?P;M?bF z6lMdxcy2pgarixG`J#7_&p@(tJU+r`GN11A-ko?rjROFJU3)ALm7I2N%MSj#RQ5~U zRG$5_cB6T+P-x_>@a>)CUF?tBfA&=F8xj>mCtyTmIplS218bHH`}
    P~(U@5t z9eRO$wPWa;sOE=Q6>!*%%ZS5WF(l!J`b#d(SiW*_osF5)(}g!g^Ew<`hUO?^%sk$- z7l6}>jOf5^Uk|3u^MXrvf~;XJe=dOkcJwopZ*<-nWvonm?Do!RB42uH3d1cOlD@5cWHkKp zaJDC9FIL(OwxpWhs0}e`dr#Ks%9hh_yyrT+?q#(UJ+@)mdF9B7#x?~T4OqNvNkZmr z-r7y-B-+5f?a6{5I?nn@`-1@Ck`wCPG<5_T$TNfi#!MY(nN+5%#h5!15c_~erl0Lj zp~#%i>S$gdBgSNzEN*pB-M21=q z3p1>{lehUiim7jUqbju)aEsRQt2dYOcdGI`TR8p-kpBnN4HR<0O|#VxaHbf`U#E-} z7QcHm#$r4$E9$tK@;}TSO(HlpJpV;V)7|{pqV&jGOc`e4#3PUhPL=OU!#cTVzWTh1 z%Hj#zXHxPAs!={J26EP01UdQmv6PgvBnmqMJWNc1{I1MXV!UE(W;e3IY=+h*Pil;*LUF%rs7ariYhREy~_VEYgylIN5 z<0SQ72}#g~Hf1%lCjNzVJI3p3u9fV8+yLel4G*V8FUoim;7KU=M4gsn4irlN6_5Z> zh!$i|fF3uT(M6Fj+nBD_gA*T8@xxend^ z;JYHW0{Nj#0*N{b68>z}gzVmPnbc9U z-9^XKin9Jx5E6H;zuhL=*D5i@!HLykjB(u7aCya|BhHO+jHbZBOq*S10iDnlit!3+ z_LpbI^mZuSlGbQE6KprWNUtGYeIO+5U9M57dx5euRL)yvsv(7-z4^4ncs9$AM5Z8LaBF0E-^C2JxXpF4;I<0(*j`o?dzr??;@(iP~0Eokj_ z{WT&ovPwYXa{tDLd?oEiSs2CaarvwfNrAAS)kwBKy$c;n<5 z-;*b=2StV6%W0Ge6j!GI&`Cfm_4~+ID^T$BB3ufdK3`@iu~0NJO#ut(kpwkL*y2yi zPKIAU%E?<1Nt}`q>>BXUDfy*pqZXk89kPdrfqZ~#83CNEnLgwM3i7t$eTtDwD3gVX zk>JVy3K?iBMvQ~AiyH?o`+JQy&_J6cW7_aUb-Zv7?zxJ}Om6iu9bA@`>5|>h>207+ zmsB=KqFVnXk%zIe=!#Hh7wOV4S7qyBwKRo=J(TfsyhJ{JK)p^FG3EiJKFgNoWtY%r zulV(+UuPj5%~?{ih7!bLF?3;={jI*~6Ew`6V>>FM*Ggk0in4xoEO{D@Gi!?7iXCgL zK7OmA=6B98j?OWRQ{Pq{D_~Ir9Zz;$gtTo^p^I9#B|gL9`8P;(Lr}l}D9(K(V3R^C z2t=0+DtgQnspV>y(??6#ML!UnJoMf^FpHfdzf&a|>>n3}bG#t&Zp+jO2nZk&@%5S( zgvkJewMGG)$ksFaGnz{z{tM%;9nTDUAjCVD-%S-ud|+B zw(B-S{Y^kKRi^TNQpGxf>Juv6{f)*onxch?!V0qCHea8`nHa% zQ#P&lNFU-S=e@sE5vVCJlnaxc!QsS4J+7febiG|pb+3^lggsuR9Xv+-g$;;r3AAU? z6HW`#XFzy04(Hf#dgP2mv=%?j)MoXbp9uMwA|QaB$fBn8=`W_`n3^<+Zl?SlRN4bD zvY9qZ!W6q#P8Z|F%Xt~85+d4SYkm>;HL2D$B$U?YBR>;%R#$+*5!R*A$cSH?OJ+e# zypG}sR`A3hC(FlG4Go$C~o>fJSBoiRbWQw7?=6$(QkelfY zbP;owtz}%0Ll)IUlVE>zulYk>KkClU{&=xyi>#qpB$Cn=V{N&G&bK1V+JLTj7NXVw zFng0WU%DG;y&J12Tevq+kAj6ffx_s7%kdp9{nsw`|;=12YtL`8!Cw|FOKLs=@{q8vycDz{}!^T8I}i z$Qk5X$5SpJ4FB@jPe_q%XypQo=Q)|9Cmc$^W*-*sZpi_Hpb=ZUW$^S!$h;mN!gQ|_ zIjnPl|9IP!*vF3%Unua56ppj7-I~zcE}HtBIBq(l<=V#MTR{>$bQNTcwDXS9=hq_I zBoswgQ?AwMdYfzq+MksUv3{>x5Hv56k_)D@>fP;y8ni9X#bC|tfyeR`l{s3|(b8nZ++yomq)cBUVn$1(+b>^#z}T%PK>>zWF^cO=g`HqS8w+vZv= z3{0CUF{=aR}N@Uft4{R z3Wq@_P_xvAO|#H==cLGSaA|*I?A?mkYHD|Un7EZVxERTBT=S#xCX|BtHswQhxF*RW z+_(xt&%h-Nv>w0KIOIyNRE!2-?E_8>A@#F9g)Z6hH)U~?l^bz(7v3x z@B!^Zu3(CQatd!q<}Db^IjuVlIyba=fsSJKb;mL4N`{LJq1W=DH^xJ(mQ>BL83S4~ z0{K$7XI z&mZvspYYFbesQ3*`J3bVufPA#Yg%j`*`xFNU!(t&r2mnXczh6#7yye)i;(o&E%`4$ zJ+^_|i+A9^S}+Oo@=NX94f%A9DP2{mvZ7ko4VLX`lV4Ui_^l7y`iL zdvhC@{5RU|<7iqM9w+~B(|7z&hWOu;uLK9=S}`Es%OwBY_}`;B0b0=j!XWs+!T==n z|3w%k4yjP@wXN#qU4iNem_cLWofpBS?zboR8jJ$ex~fOw-6U^WfR&*_IqRhzc2em{n@LeYpuQV|ge#xWIHqP32EEE;e%;qo z;*oD|XtVVX1GpjH^4pOcHzPbsAbTDNh9(Ezt@nOu;O;|t8JPG6ntpD*|MRjE$*(~G zEn)b(deK;I{^C?UuA~z(EnQPV+x*M0QEU4dE|gG83ii1MHt+GT9^NQgD2Kz5Z>(%- z_<^tkXteZb#uvyv-Z(lwV14oZv(Cc2_+?~Xlai1~kBz?J#J32ZVi_Et4&RwP$|9zF z6-BS9V5OBE_X-_&$Fz_Ksw%*I>weHTEj)oo84N*Kt0q-dRV@b#w+5QD25CRaov;&* z;`lHMe*BTI0!Ji8?V}DRd^be} z7xxY-W|D);^!>8%u9iOaGj3x`)3@$QZC{_5w;~~l|FX(&pP_|+tS$O$GeV%m{ohOz zKOrnU1#vQC)-R+j%G-aB<$nhC`vSN#oFmz-MJe+?l;z(BOB$F1*$+$qn8rVH{AH=g zBmgr7&xcF%@{fxA*2Vw%i3XTDwAa5J`oD$puYq{L1InV`iUj-nS^w);9vLOx&qDs= z=6`IkAD6b`JdbV=LJ6z&~}T+^B+_5uTcP-#Q!hiXz`iEKt>~rRT{6>0n?zb z>b`vYCzB}yhG~wHV3c|4hLsjwPJ5Z96A_Mm)ce2hq5svUlrg()78{&$Hkae8NR)Ps zrnkWV(YOD6C^}D None: + """Visualize and export experiment dependency DAGs. + + \b + Examples: + experimaestro dag # View most recent experiment in browser (default) + experimaestro dag my_experiment # Open web viewer for my_experiment + experimaestro dag my_experiment --tree # Print ASCII tree to console + experimaestro dag my_experiment --save-as dag.html # Export standalone HTML + experimaestro dag my_experiment --save-as dag.json # Export standardized JSON + experimaestro dag my_experiment --save-as dag.png # Export plot image + experimaestro dag my_experiment --list-configs # List hyperparameter trials + experimaestro dag my_experiment -c 0 # View trial #0 in browser + """ + exp = experiment_id or experiment_opt + json_out = str(json_file) if json_file else ("" if as_json else None) + + exit_code = run_dag( + experiment_id=exp, + run_id=run_id, + workspace=workspace, + web=web, + port=port, + no_open=no_open, + tree=tree, + plot=plot, + save_as=save_as, + html=html, + json_out=json_out, + tasks_only=tasks_only, + list_configs=list_configs, + single_config=single_config, + config_index=config_index, + filter_tags=filter_tags, + include_actions=include_actions, + layout=layout, + ) + if exit_code != 0: + raise click.exceptions.Exit(exit_code) diff --git a/src/experimaestro/tests/test_xpm_dag.py b/src/experimaestro/tests/test_xpm_dag.py index 7b9f9846..b9caa196 100644 --- a/src/experimaestro/tests/test_xpm_dag.py +++ b/src/experimaestro/tests/test_xpm_dag.py @@ -181,7 +181,9 @@ def test_draw_dag_cli_save_as(tmp_path, monkeypatch): draw_dag.main() assert html_out.exists() assert "" in html_out.read_text() - assert len(server_called) == 0, "Server should not be launched when --save-as ends in .html" + assert len(server_called) == 0, ( + "Server should not be launched when --save-as ends in .html" + ) # 2. Test --save-as with .json json_out = tmp_path / "subfolder" / "dag.json" @@ -210,3 +212,198 @@ def test_web_dag_template_loading(): assert "__DEFAULT_LAYOUT__" in loaded assert "__GRAPH_DATA__" in loaded + +def test_cli_dag_help(): + """Verify experimaestro dag --help and experimaestro experiments dag --help.""" + from click.testing import CliRunner + from experimaestro.cli import cli + + runner = CliRunner() + res1 = runner.invoke(cli, ["dag", "--help"]) + assert res1.exit_code == 0 + assert "Visualize and export experiment dependency DAGs" in res1.output + assert "--web" in res1.output + assert "--save-as" in res1.output + assert "--list-configs" in res1.output + + res2 = runner.invoke(cli, ["experiments", "dag", "--help"]) + assert res2.exit_code == 0 + assert "Visualize and export experiment dependency DAGs" in res2.output + + +def test_cli_dag_save_as_and_json(tmp_path, monkeypatch): + """Verify experimaestro dag --save-as and --json via Click CLI.""" + from click.testing import CliRunner + from experimaestro.cli import cli + from experimaestro.tools.xpm_dag import draw_dag + + full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()} + dependencies = {"j_train": ["j_root"]} + job_tags_dict = {"j_train": {"lr": 0.001}} + + run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000" + run_dir.mkdir(parents=True) + + monkeypatch.setattr( + draw_dag, + "resolve_run_dir", + lambda ws, exp_id, run_id: ("20260101_120000", run_dir), + ) + monkeypatch.setattr( + draw_dag, + "load_run_graph", + lambda r_dir: (full_jobs, dependencies, job_tags_dict), + ) + + runner = CliRunner() + + # 1. Test --save-as html + html_out = tmp_path / "cli_out.html" + res_html = runner.invoke(cli, ["dag", "test_exp", "--save-as", str(html_out)]) + assert res_html.exit_code == 0 + assert html_out.exists() + assert "" in html_out.read_text() + + # 2. Test --json stdout + res_json = runner.invoke(cli, ["dag", "test_exp", "--json"]) + assert res_json.exit_code == 0 + parsed = json.loads(res_json.output) + assert "nodes" in parsed + assert any(n["id"] == "j_train" for n in parsed["nodes"]) + + +def test_resolve_run_dir_auto_detect(tmp_path): + """Verify resolve_run_dir auto-selects when a single experiment exists.""" + from experimaestro.tools.xpm_dag.xpm_dag import resolve_run_dir + + ws_dir = tmp_path / "workspace" + exp_dir = ws_dir / "experiments" / "single_experiment" / "20260101_120000" + exp_dir.mkdir(parents=True) + + # 1. Single experiment exists: auto-selects without passing experiment_id + res = resolve_run_dir(workspace_arg=str(ws_dir), experiment_id=None) + assert res is not None + run_id, resolved_dir = res + assert run_id == "20260101_120000" + assert resolved_dir == exp_dir + + # 2. Multiple experiments exist: returns None when experiment_id is omitted + exp2_dir = ws_dir / "experiments" / "another_experiment" / "20260101_130000" + exp2_dir.mkdir(parents=True) + res_multi = resolve_run_dir(workspace_arg=str(ws_dir), experiment_id=None) + assert res_multi is None + + +def test_cli_dag_defaults_to_web(tmp_path, monkeypatch): + """Verify experimaestro dag defaults to launching the web viewer.""" + from click.testing import CliRunner + from experimaestro.cli import cli + from experimaestro.tools.xpm_dag import draw_dag + + full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()} + dependencies = {"j_train": ["j_root"]} + job_tags_dict = {"j_train": {"lr": 0.001}} + + run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000" + run_dir.mkdir(parents=True) + + monkeypatch.setattr( + draw_dag, + "resolve_run_dir", + lambda ws, exp_id, run_id: ("20260101_120000", run_dir), + ) + monkeypatch.setattr( + draw_dag, + "load_run_graph", + lambda r_dir: (full_jobs, dependencies, job_tags_dict), + ) + + server_called = [] + monkeypatch.setattr( + draw_dag, + "serve_dag_web", + lambda *args, **kwargs: server_called.append(True), + ) + + runner = CliRunner() + # Calling dag with NO flags: should default to web view + res = runner.invoke(cli, ["dag", "test_exp"]) + assert res.exit_code == 0 + assert len(server_called) == 1, "Web server should be launched by default" + + +def test_cli_dag_optional_tree(tmp_path, monkeypatch): + """Verify experimaestro dag --tree prints text tree and does NOT launch server.""" + from click.testing import CliRunner + from experimaestro.cli import cli + from experimaestro.tools.xpm_dag import draw_dag + + full_jobs = {"j_root": DummyTask(), "j_train": DummyTask()} + dependencies = {"j_train": ["j_root"]} + job_tags_dict = {"j_train": {"lr": 0.001}} + + run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000" + run_dir.mkdir(parents=True) + + monkeypatch.setattr( + draw_dag, + "resolve_run_dir", + lambda ws, exp_id, run_id: ("20260101_120000", run_dir), + ) + monkeypatch.setattr( + draw_dag, + "load_run_graph", + lambda r_dir: (full_jobs, dependencies, job_tags_dict), + ) + + server_called = [] + monkeypatch.setattr( + draw_dag, + "serve_dag_web", + lambda *args, **kwargs: server_called.append(True), + ) + + runner = CliRunner() + res = runner.invoke(cli, ["dag", "test_exp", "--tree"]) + assert res.exit_code == 0 + assert len(server_called) == 0, ( + "Web server should not be launched when --tree is given" + ) + assert "DummyTask" in res.output + assert "j_root" in res.output + + +def test_matplotlib_lazy_import(tmp_path, monkeypatch): + """Verify that running web view or --tree does not import matplotlib.""" + import sys + from click.testing import CliRunner + from experimaestro.cli import cli + from experimaestro.tools.xpm_dag import draw_dag + + full_jobs = {"j_root": DummyTask()} + dependencies = {} + job_tags_dict = {} + + run_dir = tmp_path / "experiments" / "test_exp" / "20260101_120000" + run_dir.mkdir(parents=True) + + monkeypatch.setattr( + draw_dag, + "resolve_run_dir", + lambda ws, exp_id, run_id: ("20260101_120000", run_dir), + ) + monkeypatch.setattr( + draw_dag, + "load_run_graph", + lambda r_dir: (full_jobs, dependencies, job_tags_dict), + ) + + # Ensure matplotlib is not in sys.modules + sys.modules.pop("matplotlib", None) + sys.modules.pop("matplotlib.pyplot", None) + + runner = CliRunner() + res = runner.invoke(cli, ["dag", "test_exp", "--tree"]) + assert res.exit_code == 0 + assert "matplotlib" not in sys.modules + assert "matplotlib.pyplot" not in sys.modules diff --git a/src/experimaestro/tools/xpm_dag/__init__.py b/src/experimaestro/tools/xpm_dag/__init__.py index f0f4e226..0cb09c8f 100644 --- a/src/experimaestro/tools/xpm_dag/__init__.py +++ b/src/experimaestro/tools/xpm_dag/__init__.py @@ -4,6 +4,7 @@ main, plot_dag, print_robust_tree, + run_dag, ) from .xpm_dag import ( BOX_H, @@ -53,5 +54,6 @@ "print_robust_tree", "resolve_run_dir", "resolve_target_config", + "run_dag", "serve_dag_web", ] diff --git a/src/experimaestro/tools/xpm_dag/draw_dag.py b/src/experimaestro/tools/xpm_dag/draw_dag.py index 95b9ab47..4c681232 100644 --- a/src/experimaestro/tools/xpm_dag/draw_dag.py +++ b/src/experimaestro/tools/xpm_dag/draw_dag.py @@ -37,6 +37,7 @@ import json import logging from pathlib import Path +import types import networkx as nx from experimaestro import tags @@ -89,6 +90,7 @@ "print_robust_tree", "resolve_run_dir", "resolve_target_config", + "run_dag", "serve_dag_web", ] @@ -123,12 +125,20 @@ def job_sort_key(jid: str): for idx, jid in enumerate(sorted_job_ids): is_last_node = is_last and (idx == len(sorted_job_ids) - 1) config = info_jobs.get(jid) - job_type = type(config).__name__.replace(".XPMConfig", "") if config else "Unknown" + job_type = ( + type(config).__name__.replace(".XPMConfig", "") if config else "Unknown" + ) depth = depths.get(jid, 0) y_coord = pos.get(jid, (0, 0))[1] # Use tags from jobs.jsonl, fallback to tags(config) - jtags = job_tags_dict.get(jid) or (tags(config) if config else {}) + jtags = job_tags_dict.get(jid) + if not jtags and config: + try: + jtags = tags(config) + except Exception: + jtags = {} + jtags = jtags or {} tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items()) label = f"{job_type} [{jid[:8]}] (depth {depth}, y {y_coord:.1f})" if tag_str: @@ -186,10 +196,18 @@ def plot_dag( show_plot: Whether to display the plot interactively. save_path: Optional path to save the figure image. """ - import matplotlib.pyplot as plt - from matplotlib.patches import FancyBboxPatch - from matplotlib.lines import Line2D - from experimaestro import Task, Action, tags + try: + import matplotlib.pyplot as plt + from matplotlib.lines import Line2D + from matplotlib.patches import FancyBboxPatch + except ImportError: + logging.error( + "matplotlib is required for graphical plotting. " + "Please install it using: pip install matplotlib" + ) + return + + from experimaestro import Action, Task, tags node_info = {} for jid, config in full_jobs.items(): @@ -197,11 +215,19 @@ def plot_dag( ntype = ( "action" if isinstance(config, Action) - else "task" if isinstance(config, Task) else "config" + else "task" + if isinstance(config, Task) + else "config" ) # Get tags and create label - jtags = job_tags_dict.get(jid) or (tags(config) if config else {}) + jtags = job_tags_dict.get(jid) + if not jtags and config: + try: + jtags = tags(config) + except Exception: + jtags = {} + jtags = jtags or {} tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items()) node_info[jid] = { @@ -346,245 +372,357 @@ def plot_dag( plt.close() -def main() -> None: - logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--workspace", - type=str, - default=None, - help="Workspace ID or directory path (default: auto-detected via find_workspace).", - ) - parser.add_argument( - "--experiment-id", - default="MNIST_train_advanced", - help="Experiment identifier (default: %(default)s).", - ) - parser.add_argument( - "--run-id", - default=None, - help="Specific run timestamp ID (default: most recent run).", - ) - parser.add_argument( - "--tasks-only", - action="store_true", - help="Only display Task and Action nodes, skipping static config nodes.", - ) - parser.add_argument( - "--web", - action="store_true", - help="Launch interactive DAG web viewer and open in browser.", - ) - parser.add_argument( - "--html", - type=Path, - default=None, - help="Path where the standalone interactive DAG HTML page should be saved (e.g. dag.html).", - ) - parser.add_argument( - "--json", - nargs="?", - const="", - default=None, - help="Export DAG as JSON to stdout or file (compatible with Experimaestro web CLI).", - ) - parser.add_argument( - "--no-plot", - action="store_false", - dest="plot", - default=True, - help="Skip the matplotlib interactive plot and only output the text tree.", - ) - parser.add_argument( - "--list-configs", - action="store_true", - help="List all unique configuration parameter sets found in the experiment and exit.", - ) - parser.add_argument( - "--single-config", - action="store_true", - help="Render the DAG for a single grid configuration (defaults to config index 0).", - ) - parser.add_argument( - "-c", - "--config-index", - type=int, - default=None, - help="Index of the configuration to display (see --list-configs). Implies --single-config.", - ) - parser.add_argument( - "--filter", - default=None, - help="Filter the DAG by tag values (e.g. 'lr=0.001,kernel_size=3').", - ) - parser.add_argument( - "--include-actions", - action="store_true", - help="Include downstream Action and EvaluatedModel nodes when filtering by configuration.", - ) - parser.add_argument( - "--layout", - choices=["dagre", "aligned"], - default="dagre", - help="Layout algorithm for interactive web view: 'dagre' (default) or 'aligned' (stage-aligned flow).", - ) - parser.add_argument( - "--save-as", - type=Path, - default=None, - help="Path where the DAG output should be saved (auto-detects .html, .json, .png, etc.). " - "If ending in .html, generates standalone HTML without launching the server.", - ) - args = parser.parse_args() - +def run_dag( + *, + experiment_id: str | None = None, + run_id: str | None = None, + workspace: str | None = None, + web: bool | None = None, + port: int = 0, + no_open: bool = False, + tree: bool = False, + plot: bool = False, + save_as: Path | None = None, + html: Path | None = None, + json_out: str | None = None, + tasks_only: bool = False, + list_configs: bool = False, + single_config: bool = False, + config_index: int | None = None, + filter_tags: str | None = None, + include_actions: bool = False, + layout: str = "dagre", +) -> int: + """Execute the experiment DAG visualization/export pipeline with given options.""" save_plot_path = None - if args.save_as: - ext = args.save_as.suffix.lower() + if save_as: + ext = save_as.suffix.lower() if ext in (".html", ".htm"): - args.html = args.save_as - args.web = False + html = save_as + web = False elif ext == ".json": - args.json = str(args.save_as) - args.web = False + json_out = str(save_as) + web = False elif ext in (".png", ".pdf", ".svg", ".jpg", ".jpeg"): - save_plot_path = args.save_as - args.plot = False - args.web = False + save_plot_path = save_as + plot = False + web = False else: logging.error( "Unsupported file extension '%s' for --save-as. " "Supported extensions: .html, .htm, .json, .png, .pdf, .svg, .jpg", ext, ) - return + return 1 + + # Resolve default web behavior: enabled by default unless --tree, --plot, or file export is active + if web is None: + web = not ( + plot or tree or save_as or html or json_out is not None or list_configs + ) - run_info = resolve_run_dir(args.workspace, args.experiment_id, args.run_id) + run_info = resolve_run_dir(workspace, experiment_id, run_id) if run_info is None: - return + return 1 run_id, run_dir = run_info + exp_id = run_dir.parent.name - if args.list_configs: + if list_configs: configs = get_experiment_configs(run_dir) if not configs: - print(f"No configuration tags found for experiment '{args.experiment_id}' ({run_id}).") # noqa: T201 - return - print(f"Available configurations in experiment '{args.experiment_id}' (run {run_id}):") # noqa: T201 + print(f"No configuration tags found for experiment '{exp_id}' ({run_id}).") # noqa: T201 + return 0 + print(f"Available configurations in experiment '{exp_id}' (run {run_id}):") # noqa: T201 for idx, cfg_dict in enumerate(configs): cfg_str = ", ".join(f"{k}={v}" for k, v in sorted(cfg_dict.items())) print(f" [{idx}] {cfg_str}") # noqa: T201 - return + return 0 graph_data = load_run_graph(run_dir) if graph_data is None: - return + return 1 full_jobs, dependencies, job_tags_dict = graph_data title_suffix = "" - if args.single_config or args.config_index is not None or args.filter: + if single_config or config_index is not None or filter_tags: configs = get_experiment_configs(run_dir) - target_tags, title_suffix = resolve_target_config(args, configs) + filter_args = types.SimpleNamespace( + config_index=config_index, + filter=filter_tags, + single_config=single_config, + ) + target_tags, title_suffix = resolve_target_config(filter_args, configs) if target_tags is None: - return + return 1 full_jobs, dependencies = filter_subgraph_by_config( full_jobs, dependencies, job_tags_dict, target_tags, - include_actions=args.include_actions, + include_actions=include_actions, ) - if args.tasks_only: + if tasks_only: full_jobs, dependencies = filter_tasks_only(full_jobs, dependencies) workspace_path = str(run_dir.parents[2]) if len(run_dir.parents) >= 3 else "" # Export JSON if requested - if args.json is not None: + if json_out is not None: dag_json = build_dag_json( full_jobs, dependencies, job_tags_dict, run_dir=run_dir, - experiment_id=args.experiment_id, + experiment_id=exp_id, run_id=run_id, workspace=workspace_path, ) json_str = json.dumps(dag_json, indent=2) - if args.json: # Target file provided - out_path = Path(args.json) + if json_out: # Target file provided + out_path = Path(json_out) out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(json_str) print(f"Saved DAG JSON to {out_path}") # noqa: T201 else: print(json_str) # noqa: T201 - return + return 0 # Export standalone HTML page if requested - if args.html is not None: + if html is not None: dag_json = build_dag_json( full_jobs, dependencies, job_tags_dict, run_dir=run_dir, - experiment_id=args.experiment_id, + experiment_id=exp_id, run_id=run_id, workspace=workspace_path, ) html_str = generate_dag_html( dag_json, - title=f"Experimaestro DAG - {args.experiment_id}", - default_layout=args.layout, + title=f"Experimaestro DAG - {exp_id}", + default_layout=layout, ) - args.html.parent.mkdir(parents=True, exist_ok=True) - args.html.write_text(html_str) - print(f"Saved interactive DAG HTML to {args.html}") # noqa: T201 - return + html.parent.mkdir(parents=True, exist_ok=True) + html.write_text(html_str) + print(f"Saved interactive DAG HTML to {html}") # noqa: T201 + return 0 + + # Optional terminal text tree visualization + if tree: + dependents: dict[str, list[str]] = {} + for jid, deps in dependencies.items(): + for d in deps: + dependents.setdefault(d, []).append(jid) + + depths, pos, _ = compute_dag_layout(full_jobs, dependencies, dependents) + + tree_roots = [jid for jid in full_jobs if not dependencies.get(jid)] + print_robust_tree( + tree_roots, + dependents, + dependencies, + full_jobs, + job_tags_dict, + set(), + depths, + pos, + ) + if not (web or plot or save_plot_path): + return 0 - # Launch interactive web server if requested - if args.web: + # Launch interactive web server (default mode) + if web: dag_json = build_dag_json( full_jobs, dependencies, job_tags_dict, run_dir=run_dir, - experiment_id=args.experiment_id, + experiment_id=exp_id, run_id=run_id, workspace=workspace_path, ) - serve_dag_web(dag_json, default_layout=args.layout) - return + serve_dag_web( + dag_json, + port=port, + open_browser=not no_open, + default_layout=layout, + ) + return 0 - # Default / Matplotlib & Text Tree visualization - dependents: dict[str, list[str]] = {} - for jid, deps in dependencies.items(): - for d in deps: - dependents.setdefault(d, []).append(jid) - - depths, pos, reduced_dag = compute_dag_layout(full_jobs, dependencies, dependents) - - tree_roots = [jid for jid in full_jobs if not dependencies.get(jid)] - print_robust_tree( - tree_roots, - dependents, - dependencies, - full_jobs, - job_tags_dict, - set(), - depths, - pos, - ) + # Optional Matplotlib & graphical plot visualization (requires matplotlib) + if plot or save_plot_path: + dependents = {} + for jid, deps in dependencies.items(): + for d in deps: + dependents.setdefault(d, []).append(jid) + + depths, pos, reduced_dag = compute_dag_layout( + full_jobs, dependencies, dependents + ) - if args.plot or save_plot_path: plot_dag( full_jobs, job_tags_dict, pos, reduced_dag, title_suffix=title_suffix, - show_plot=args.plot, + show_plot=plot, save_path=save_plot_path, ) + return 0 + + return 0 + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "experiment", + nargs="?", + default=None, + help="Experiment identifier (optional positional).", + ) + parser.add_argument( + "--workspace", + "-w", + type=str, + default=None, + help="Workspace ID or directory path (default: auto-detected via find_workspace).", + ) + parser.add_argument( + "--experiment-id", + "-e", + default=None, + help="Experiment identifier.", + ) + parser.add_argument( + "--run-id", + "-r", + default=None, + help="Specific run timestamp ID (default: most recent run).", + ) + parser.add_argument( + "--tasks-only", + action="store_true", + help="Only display Task and Action nodes, skipping static config nodes.", + ) + parser.add_argument( + "--web", + "-W", + action="store_true", + default=None, + help="Launch interactive DAG web viewer in browser (default).", + ) + parser.add_argument( + "--no-web", + action="store_false", + dest="web", + help="Do not launch interactive DAG web viewer.", + ) + parser.add_argument( + "--tree", + "-t", + action="store_true", + help="Print text tree representation of the DAG to terminal.", + ) + parser.add_argument( + "--plot", + action="store_true", + default=False, + help="Display matplotlib interactive plot window (requires matplotlib).", + ) + parser.add_argument( + "--port", + type=int, + default=0, + help="Port for web server (default: random available port).", + ) + parser.add_argument( + "--no-open", + action="store_true", + help="Do not automatically open browser when launching web viewer.", + ) + parser.add_argument( + "--html", + type=Path, + default=None, + help="Path where the standalone interactive DAG HTML page should be saved (e.g. dag.html).", + ) + parser.add_argument( + "--json", + nargs="?", + const="", + default=None, + help="Export DAG as JSON to stdout or file (compatible with Experimaestro web CLI).", + ) + parser.add_argument( + "--list-configs", + action="store_true", + help="List all unique configuration parameter sets found in the experiment and exit.", + ) + parser.add_argument( + "--single-config", + action="store_true", + help="Render the DAG for a single grid configuration (defaults to config index 0).", + ) + parser.add_argument( + "-c", + "--config-index", + type=int, + default=None, + help="Index of the configuration to display (see --list-configs). Implies --single-config.", + ) + parser.add_argument( + "--filter", + default=None, + help="Filter the DAG by tag values (e.g. 'lr=0.001,kernel_size=3').", + ) + parser.add_argument( + "--include-actions", + action="store_true", + help="Include downstream Action and EvaluatedModel nodes when filtering by configuration.", + ) + parser.add_argument( + "--layout", + choices=["dagre", "aligned"], + default="dagre", + help="Layout algorithm for interactive web view: 'dagre' (default) or 'aligned' (stage-aligned flow).", + ) + parser.add_argument( + "--save-as", + type=Path, + default=None, + help="Path where the DAG output should be saved (auto-detects .html, .json, .png, etc.). " + "If ending in .html, generates standalone HTML without launching the server.", + ) + args = parser.parse_args() + + exp_id = args.experiment or args.experiment_id + + run_dag( + experiment_id=exp_id, + run_id=args.run_id, + workspace=args.workspace, + web=args.web, + port=args.port, + no_open=args.no_open, + tree=args.tree, + plot=args.plot, + save_as=args.save_as, + html=args.html, + json_out=args.json, + tasks_only=args.tasks_only, + list_configs=args.list_configs, + single_config=args.single_config, + config_index=args.config_index, + filter_tags=args.filter, + include_actions=args.include_actions, + layout=args.layout, + ) if __name__ == "__main__": diff --git a/src/experimaestro/tools/xpm_dag/xpm_dag.py b/src/experimaestro/tools/xpm_dag/xpm_dag.py index 03cd759a..cff71845 100644 --- a/src/experimaestro/tools/xpm_dag/xpm_dag.py +++ b/src/experimaestro/tools/xpm_dag/xpm_dag.py @@ -30,7 +30,9 @@ def resolve_run_dir( - workspace_arg: str | Path | None, experiment_id: str, run_id: str | None + workspace_arg: str | Path | None, + experiment_id: str | None = None, + run_id: str | None = None, ) -> tuple[str, Path] | None: """Find and return (run_id, run_dir) from the workspace. @@ -55,11 +57,37 @@ def resolve_run_dir( return None available_xps = sorted( - [p.name for p in experiments_dir.iterdir() if p.is_dir() and not p.name.startswith(".")] + [ + p.name + for p in experiments_dir.iterdir() + if p.is_dir() and not p.name.startswith(".") + ] ) + if not experiment_id: + if len(available_xps) == 1: + experiment_id = available_xps[0] + logging.info( + "Auto-selected single experiment '%s' in workspace '%s'.", + experiment_id, + workspace, + ) + elif len(available_xps) > 1: + xps_str = ", ".join(f"'{x}'" for x in available_xps) + logging.error( + "No experiment specified. Please specify an experiment ID. Available experiments in workspace '%s': %s", + workspace, + xps_str, + ) + return None + else: + logging.error("No experiments found in workspace '%s'.", workspace) + return None + exp_dir = experiments_dir / experiment_id if not exp_dir.exists(): - xps_str = ", ".join(f"'{x}'" for x in available_xps) if available_xps else "(none)" + xps_str = ( + ", ".join(f"'{x}'" for x in available_xps) if available_xps else "(none)" + ) logging.error( "Experiment '%s' not found in workspace '%s'. Available experiments: %s", experiment_id, @@ -339,7 +367,9 @@ def filter_subgraph_by_config( from experimaestro import Action for jid, config in full_jobs.items(): - if isinstance(config, Action) or type(config).__name__.startswith("EvaluatedModel"): + if isinstance(config, Action) or type(config).__name__.startswith( + "EvaluatedModel" + ): deps = dependencies.get(jid, []) if any(m in deps for m in matching): selected_nodes.add(jid) @@ -415,8 +445,7 @@ def compute_dag_depths( nodes_t = [ n for n in dag.nodes - if n in full_jobs - and type(full_jobs[n]).__name__.split(".")[0] == tname + if n in full_jobs and type(full_jobs[n]).__name__.split(".")[0] == tname ] if nodes_t: max_d = max(depths[n] for n in nodes_t) @@ -497,14 +526,18 @@ def resolve_target_config( ) return None, "" target_tags = configs[args.config_index] - return target_tags, ", ".join(f"{k}={v}" for k, v in sorted(target_tags.items())) + return target_tags, ", ".join( + f"{k}={v}" for k, v in sorted(target_tags.items()) + ) if args.filter: filter_tags = parse_filter_arg(args.filter) if args.single_config: for cfg_cand in configs: if all(cfg_cand.get(k) == v for k, v in filter_tags.items()): - return cfg_cand, ", ".join(f"{k}={v}" for k, v in sorted(cfg_cand.items())) + return cfg_cand, ", ".join( + f"{k}={v}" for k, v in sorted(cfg_cand.items()) + ) logging.error("No configuration matched filter: %s", args.filter) return None, "" return filter_tags, f"Filter: {args.filter}" From fbc941d8bd1545daacddcd68d5e8da7b19005425 Mon Sep 17 00:00:00 2001 From: Victor Morand Date: Mon, 19 Oct 2026 14:50:48 +0200 Subject: [PATCH 3/7] feat: inspect params tool --- docs/source/cli.md | 43 + docs/source/demo | 2 +- src/experimaestro/cli/__init__.py | 5 + src/experimaestro/cli/inspect.py | 58 + .../tests/test_inspect_params.py | 874 ++++++++++++++ src/experimaestro/tests/test_xpm_dag.py | 151 +++ .../tools/inspect_params/__init__.py | 69 ++ src/experimaestro/tools/inspect_params/app.py | 1063 +++++++++++++++++ .../tools/inspect_params/discovery.py | 257 ++++ .../tools/inspect_params/models.py | 321 +++++ src/experimaestro/tools/xpm_dag/xpm_dag.py | 9 + src/experimaestro/utils/remote.py | 383 ++++++ 12 files changed, 3234 insertions(+), 1 deletion(-) create mode 100644 src/experimaestro/cli/inspect.py create mode 100644 src/experimaestro/tests/test_inspect_params.py create mode 100644 src/experimaestro/tools/inspect_params/__init__.py create mode 100644 src/experimaestro/tools/inspect_params/app.py create mode 100644 src/experimaestro/tools/inspect_params/discovery.py create mode 100644 src/experimaestro/tools/inspect_params/models.py create mode 100644 src/experimaestro/utils/remote.py diff --git a/docs/source/cli.md b/docs/source/cli.md index c7111075..94ff590e 100644 --- a/docs/source/cli.md +++ b/docs/source/cli.md @@ -131,6 +131,9 @@ experimaestro jobs log -f mymodule.MyTask/abc123def # Get job directory path experimaestro jobs path mymodule.MyTask/abc123def + +# Inspect job parameters in interactive TUI +experimaestro inspect mymodule.MyTask/abc123def ``` ## Filter Expressions @@ -540,3 +543,43 @@ experimaestro dag my_experiment --json | `--include-actions` | Include downstream Action and EvaluatedModel nodes when filtering configs | | `--layout` | Web viewer layout algorithm: `dagre` (default) or `aligned` | +## Job Parameters Inspector (`experimaestro inspect`) + +Experimaestro provides an interactive terminal-based (TUI) parameters inspector built with Textual. It visualizes the hierarchical configuration graph of jobs, with support for cycle detection, relative path resolution, Python code synthesis, and remote workspace job inspection via SSH. + +```bash +# Inspect current directory / job +experimaestro inspect + +# Inspect a specific job by task name or hash prefix +experimaestro inspect mymodule.MyTask/1822b2b + +# Inspect a specific params.json directly +experimaestro inspect /path/to/params.json + +# Inspect a job in a remote SSH workspace +experimaestro inspect -w my_cluster mymodule.MyTask/1822b2b +``` + +### Options + +| Option | Description | +|--------|-------------| +| `TARGET` | Job ID or hash prefix, path to `params.json` / `experimaestro.json`, or job directory (default: `.`) | +| `-w, --workspace` | Workspace ID or directory path (defaults to current Experimaestro workspace; supports remote SSH workspaces) | + +### Keybindings + +| Key | Action | +|-----|--------| +| `q` / `Esc` | Quit / close active search or modal | +| `/` | Focus search bar to filter nodes by parameter name or value | +| `g` | Toggle parameter grouping (alphabetical vs categorized) | +| `p` | Toggle path display (relative vs absolute) | +| `c` | Copy synthesized Python code snippet to clipboard | +| `s` | Copy selected parameter value or resolved path to clipboard | +| `J` | Toggle raw JSON view | +| `e` | Expand all nodes | +| `c` (on parent) | Collapse all child nodes | +| `?` | Show interactive keyboard shortcuts and help | + diff --git a/docs/source/demo b/docs/source/demo index 25283de3..ae58e308 160000 --- a/docs/source/demo +++ b/docs/source/demo @@ -1 +1 @@ -Subproject commit 25283de340edb9254351829372708bef3a1ee059 +Subproject commit ae58e3085047858f36e86bfbc0ef504768055e56 diff --git a/src/experimaestro/cli/__init__.py b/src/experimaestro/cli/__init__.py index 2c41f1cf..4d4d05a1 100644 --- a/src/experimaestro/cli/__init__.py +++ b/src/experimaestro/cli/__init__.py @@ -329,6 +329,11 @@ def get_command(self, ctx, name): cli.add_command(dag_cli) +# Import and add inspect command +from .inspect import inspect_command # noqa: E402 + +cli.add_command(inspect_command, "inspect") + @cli.group() def migrate(): diff --git a/src/experimaestro/cli/inspect.py b/src/experimaestro/cli/inspect.py new file mode 100644 index 00000000..f2033f58 --- /dev/null +++ b/src/experimaestro/cli/inspect.py @@ -0,0 +1,58 @@ +"""Click CLI command for Experimaestro Parameters Inspector (`experimaestro inspect`).""" + +from __future__ import annotations + +import sys +from typing import Optional + +import click + + +@click.command(name="inspect") +@click.argument("target", required=False, default=".") +@click.option( + "-w", + "--workspace", + default=None, + help="Workspace ID or directory path (defaults to current Experimaestro workspace; supports remote SSH workspaces).", +) +def inspect_command(target: str, workspace: Optional[str]) -> None: + """Interactive TUI Parameters Inspector for Experimaestro jobs. + + TARGET can be a Job ID or hash prefix (e.g., `mymodule.MyTask/1822b2b`), + a path to a `params.json` or `experimaestro.json` file, or a job directory. + Defaults to `.` (current directory or default workspace). + + Examples: + + \b + # Inspect current directory / job + experimaestro inspect + + \b + # Inspect a specific job by task name or hash prefix + experimaestro inspect [mymodule.MyTask/]1822b2b + + \b + # Inspect a specific params.json directly + experimaestro inspect /path/to/params.json + + \b + # Inspect a job in a specific workspace (workspace can be local or remote) + experimaestro inspect -w my_(remote)_workspace [mymodule.MyTask/]1822b2b + """ + from experimaestro.tools.inspect_params import ( + ParamsInspectorApp, + find_params_file, + ) + + try: + params_path = find_params_file(target, workspace) + app = ParamsInspectorApp(params_path) + app.run() + except FileNotFoundError as e: + click.echo(f"Error: {e}", err=True) + sys.exit(1) + except Exception as e: + click.echo(f"Error running parameters inspector: {e}", err=True) + sys.exit(1) diff --git a/src/experimaestro/tests/test_inspect_params.py b/src/experimaestro/tests/test_inspect_params.py new file mode 100644 index 00000000..8eb04697 --- /dev/null +++ b/src/experimaestro/tests/test_inspect_params.py @@ -0,0 +1,874 @@ +"""Automated tests for the standalone Params Inspector tool.""" + +from pathlib import Path +import json +import pytest + +# Ensure demo directory is on sys.path + +from experimaestro.settings import SSHSettings, WorkspaceSettings +from experimaestro.tools.inspect_params import ( + NodeKind, + NodeDetailPane, + ParamsGraph, + ParamsInspectorApp, + ParamsTree, + ResolvedPath, + clean_job_target, + fetch_remote_workspace_file, + find_local_workspace_file, + find_params_file, + find_workspace_file, + resolve_job_file_pattern, + resolve_workspace, +) + +MNIST_PARAMS_PATH = Path( + "/Users/victor/code/experiments/cross_encoders/jobs/mnist_xp.learn.learn/1822b2bdcb83480cfe41b9dafcc66ceb6d32aef20ca2d64ed703da906da1d028/params.json" +) + +EVALUATE_PARAMS_PATH = Path( + "/Users/victor/code/experiments/cross_encoders/jobs/mnist_xp.learn.evaluate/4379be85e1b9430d4e72a52546903d83095711707b8fd05e05024325809c23be/params.json" +) + + +@pytest.fixture(autouse=True) +def setup_test_workspaces(): + from experimaestro.settings import get_settings, WorkspaceSettings + + settings = get_settings() + ws_path = Path("/Users/victor/code/experiments/cross_encoders") + if not any(w.id == "cross_encoders" for w in settings.workspaces): + settings.workspaces.append(WorkspaceSettings(id="cross_encoders", path=ws_path)) + yield + + +def test_mnist_params_graph_parsing(): + """Verify parsing and reference graph construction on MNIST learn job.""" + assert MNIST_PARAMS_PATH.exists(), f"Test fixture not found: {MNIST_PARAMS_PATH}" + graph = ParamsGraph.from_file(MNIST_PARAMS_PATH) + + assert graph.version == 3 + assert len(graph.objects) == 3 + + # Root task should be Learn + assert graph.root_task_id is not None + root = graph.objects[graph.root_task_id] + assert root.type_name == "Learn" + assert root.is_root_task is True + assert root.is_task is True + assert ( + root.identifier + == "1822b2bdcb83480cfe41b9dafcc66ceb6d32aef20ca2d64ed703da906da1d028" + ) + + # Outgoing references + out_map = dict(root.outgoing_refs) + assert "data" in out_map + assert "model" in out_map + + # Check model (CNN) + cnn_id = out_map["model"] + cnn_obj = graph.objects[cnn_id] + assert cnn_obj.type_name == "CNN" + assert cnn_obj.fields["hidden_dim"] == 2 + assert cnn_obj.fields["kernel_size"] == 3 + assert (root.id, "model") in cnn_obj.incoming_refs + + # Check path resolution + run_path_res = graph.resolve_path(root.fields["run_path"]) + assert isinstance(run_path_res, ResolvedPath) + assert run_path_res.base == "job" + assert run_path_res.raw_value == "runs" + assert run_path_res.resolved == MNIST_PARAMS_PATH.parent / "runs" + assert run_path_res.exists is True + assert run_path_res.is_dir is True + + # Python code synthesis + py_code = graph.to_python_snippet(root.id) + assert "Learn.C(" in py_code + assert "CNN.C(" in py_code + assert "batch_size=64" in py_code + assert "lr=0.001" in py_code + + +def test_evaluate_params_with_init_tasks_and_shared_nodes(): + """Verify parsing on evaluate job with init-tasks and shared CNN config.""" + assert EVALUATE_PARAMS_PATH.exists(), ( + f"Test fixture not found: {EVALUATE_PARAMS_PATH}" + ) + graph = ParamsGraph.from_file(EVALUATE_PARAMS_PATH) + + assert graph.root_task_id is not None + root = graph.objects[graph.root_task_id] + assert root.type_name == "Evaluate" + assert len(root.init_tasks) == 1 + + init_task_id = root.init_tasks[0] + init_task_node = graph.objects[init_task_id] + assert init_task_node.type_name == "ParameterLoader" + + # CNN model should be shared between Evaluate and ParameterLoader + cnn_id = None + for k, v in root.fields.items(): + if k == "model": + cnn_id = int(v["value"]) + assert cnn_id is not None + cnn_node = graph.objects[cnn_id] + assert cnn_node.type_name == "CNN" + assert len(cnn_node.incoming_refs) >= 2 + + +def test_cycle_detection(): + """Verify that circular references are safely handled without infinite recursion.""" + cycle_data = { + "workspace": "/tmp/test", + "version": 3, + "experimaestro": "test", + "objects": [ + { + "id": 101, + "module": "test.pkg", + "type": "ConfigA", + "identifier": "aaaa" * 16, + "fields": { + "next": {"type": "python", "value": 102}, + "val": 42, + }, + }, + { + "id": 102, + "module": "test.pkg", + "type": "ConfigB", + "identifier": "bbbb" * 16, + "fields": { + "loop_back": {"type": "python", "value": 101}, + "name": "hello", + }, + }, + ], + } + + graph = ParamsGraph(cycle_data, job_path=Path("/tmp/test/job")) + assert len(graph.objects) == 2 + assert graph.root_task_id == 102 + + # Code generator should not crash on cycle + code = graph.to_python_snippet(101) + assert "# cycle to #101" in code + + # Tree should render cycle node cleanly + tree = ParamsTree(graph) + tree.show_root = False + tree.guide_depth = 3 + tree.rebuild() + + # Look for cycle node + cycle_nodes = [] + + def find_cycle(node): + if node.data and node.data.kind == NodeKind.CYCLE: + cycle_nodes.append(node) + for c in node.children: + find_cycle(c) + + find_cycle(tree.root) + assert len(cycle_nodes) > 0, "Expected a cycle node to be rendered" + assert cycle_nodes[0].data.cycle_chain == [102, 101, 102] + + +@pytest.mark.anyio +async def test_app_pilot_smoke_test(): + """Headless pilot test of ParamsInspectorApp.""" + app = ParamsInspectorApp(MNIST_PARAMS_PATH) + async with app.run_test() as pilot: + await pilot.pause() + + # Check title and subtitle + assert "Experimaestro Parameters Inspector" in app.title + + tree = app.query_one(ParamsTree) + detail = app.query_one(NodeDetailPane) + + # Verify initial tree state (categorized grouping) + assert tree.group_fields is True + assert detail.show_raw_json is False + + # Toggle field grouping (Alphabetical vs Categorized) + await pilot.press("g") + await pilot.pause() + assert tree.group_fields is False + + await pilot.press("g") + await pilot.pause() + assert tree.group_fields is True + + # Toggle paths (Relative vs Absolute) + await pilot.press("p") + await pilot.pause() + assert tree.show_absolute_paths is True + + # Toggle Raw JSON view in detail pane + await pilot.press("J") + await pilot.pause() + assert detail.show_raw_json is True + + await pilot.press("J") + await pilot.pause() + assert detail.show_raw_json is False + + # Toggle search bar + await pilot.press("slash") + await pilot.pause() + search_box = app.query_one("#search-bar-container") + assert "hidden" not in search_box.classes + + # Close search bar with escape + await pilot.press("escape") + await pilot.pause() + assert "hidden" in search_box.classes + + +@pytest.mark.anyio +async def test_app_pilot_jump_unfolds_config(): + """Verify that jumping to primary definition with 'j' unfolds folded ancestors and config.""" + app = ParamsInspectorApp(EVALUATE_PARAMS_PATH) + async with app.run_test() as pilot: + await pilot.pause() + tree = app.query_one(ParamsTree) + app.action_expand_all() + await pilot.pause() + + # Find shared node pointing to CNN + shared_node = None + + def find_shared(n): + nonlocal shared_node + if n.data and n.data.target_object_id: + shared_node = n + return + for c in n.children: + find_shared(c) + + find_shared(tree.root) + assert shared_node is not None + + tid = shared_node.data.target_object_id + primary = tree.primary_nodes[tid] + + # Explicitly collapse primary and its parent category + primary.collapse() + primary.parent.collapse() + await pilot.pause() + assert primary.is_expanded is False + assert primary.parent.is_expanded is False + + # Move cursor to shared node + tree.move_cursor(shared_node) + await pilot.pause() + assert tree.cursor_node == shared_node + + # Press 'j' to jump + await pilot.press("j") + await pilot.pause() + + # Verify cursor jumped to primary, and both parent and primary are unfolded + assert tree.cursor_node == primary + assert primary.parent.is_expanded is True + assert primary.is_expanded is True + + # Re-collapse and test that Enter on a shared node also jumps and unfolds + primary.collapse() + primary.parent.collapse() + await pilot.pause() + assert primary.is_expanded is False + + tree.move_cursor(shared_node) + await pilot.pause() + await pilot.press("enter") + await pilot.pause() + + assert tree.cursor_node == primary + assert primary.parent.is_expanded is True + assert primary.is_expanded is True + + +def test_find_params_file_absolute_path(): + """Verify that absolute paths resolve directly to file or directory.""" + # Absolute path to params.json + res1 = find_params_file(str(MNIST_PARAMS_PATH)) + assert res1 == MNIST_PARAMS_PATH + + # Absolute path with explicit local workspace specified must NOT load from workspace + res1_ws = find_params_file(str(MNIST_PARAMS_PATH), workspace="cross_encoders") + assert res1_ws == MNIST_PARAMS_PATH + + # Absolute path to parent job directory + res2 = find_params_file(str(MNIST_PARAMS_PATH.parent)) + assert res2 == MNIST_PARAMS_PATH + + # Non-existent absolute path + with pytest.raises( + FileNotFoundError, match="params.json not found at absolute path" + ): + find_params_file("/tmp/nonexistent_dir_12345/params.json") + + +def test_find_params_file_job_id_default_workspace(caplog): + """Verify that job ID prefix resolves via default workspace discovery and logs it.""" + import logging + + with caplog.at_level(logging.INFO): + ws = resolve_workspace() + assert ws.exists() + res = find_params_file("1822b2b") + assert res == MNIST_PARAMS_PATH + assert "Using default workspace" in caplog.text + + +def test_find_params_file_explicit_workspace(caplog): + """Verify resolving job via explicit workspace name/path and logging it.""" + import logging + + # By workspace name + with caplog.at_level(logging.INFO): + res1 = find_params_file("4379be8", workspace="cross_encoders") + assert res1 == EVALUATE_PARAMS_PATH + assert "Using workspace cross_encoders" in caplog.text + + # By workspace path + with caplog.at_level(logging.INFO): + res2 = find_params_file( + "4379be8", workspace="/Users/victor/code/experiments/cross_encoders" + ) + assert res2 == EVALUATE_PARAMS_PATH + assert ( + "Using workspace /Users/victor/code/experiments/cross_encoders" + in caplog.text + ) + + +def test_find_params_file_multiple_matches_warning(capsys): + """Verify warning when multiple jobs match a short prefix.""" + res = find_params_file("1", workspace="cross_encoders") + captured = capsys.readouterr() + assert "Warning: Multiple jobs" in captured.err + assert "Using the first match:" in captured.err + assert res.exists() + assert res.name == "params.json" + + +def test_find_params_file_not_found(): + """Verify FileNotFoundError when job ID does not exist.""" + with pytest.raises( + FileNotFoundError, match="No job matching 'nonexistent_job_prefix_xyz' found" + ): + find_params_file("nonexistent_job_prefix_xyz", workspace="cross_encoders") + + +def make_mock_remote_workspace() -> WorkspaceSettings: + return WorkspaceSettings( + id="mock-remote-ws", + path=Path("/remote/workspace/path"), + ssh=SSHSettings(host="remote-host", options=["-o", "StrictHostKeyChecking=no"]), + ) + + +def test_clean_job_target(): + """Verify target normalization with various prefixes and paths.""" + assert clean_job_target("ee76fb8f") == "ee76fb8f" + assert clean_job_target("my_task/ee76fb8f") == "my_task/ee76fb8f" + assert clean_job_target("jobs/my_task/ee76fb8f") == "my_task/ee76fb8f" + assert clean_job_target("/jobs/my_task/ee76fb8f") == "my_task/ee76fb8f" + assert ( + clean_job_target( + "/remote/ws/jobs/my_task/ee76fb8f/params.json", + ws_base="/remote/ws", + filename="params.json", + ) + == "my_task/ee76fb8f" + ) + assert ( + clean_job_target( + "my_task/ee76fb8f/experimaestro.json", filename="experimaestro.json" + ) + == "my_task/ee76fb8f" + ) + with pytest.raises(ValueError): + clean_job_target("") + with pytest.raises(ValueError): + clean_job_target(".") + + +def test_resolve_job_file_pattern(): + """Verify relative glob pattern construction.""" + assert ( + resolve_job_file_pattern("ee76fb8f", "params.json") + == "jobs/*/ee76fb8f*/params.json" + ) + assert ( + resolve_job_file_pattern("my_task/ee76fb8f", "params.json") + == "jobs/my_task/ee76fb8f*/params.json" + ) + assert ( + resolve_job_file_pattern("my_task:ee76fb8f", "experimaestro.json") + == "jobs/my_task/ee76fb8f*/experimaestro.json" + ) + + +def test_find_local_workspace_file_success(tmp_path): + """Verify finding files within a local workspace directory.""" + job_dir = tmp_path / "jobs" / "my_task" / "ee76fb8f12345" + job_dir.mkdir(parents=True) + params_file = job_dir / "params.json" + params_file.write_text("{}", encoding="utf-8") + xp_file = job_dir / "experimaestro.json" + xp_file.write_text("[]", encoding="utf-8") + + # Find params.json by prefix + found = find_local_workspace_file(tmp_path, "ee76fb8f") + assert found == params_file.resolve() + + # Find custom filename by task/prefix + found_xp = find_local_workspace_file( + tmp_path, "my_task/ee76", filename="experimaestro.json" + ) + assert found_xp == xp_file.resolve() + + +def test_find_local_workspace_file_not_found(tmp_path): + """Verify FileNotFoundError when local workspace has no matching job file.""" + with pytest.raises(FileNotFoundError, match="No job matching 'nonexistent' found"): + find_local_workspace_file(tmp_path, "nonexistent") + + +def test_fetch_remote_workspace_file_success(tmp_path, monkeypatch): + """Verify remote file discovery via SSH and synchronization via rsync.""" + ws = make_mock_remote_workspace() + target_job = "ee76fb8f" + remote_matched_path = f"{ws._raw_path}/jobs/my_task/{target_job}12345/params.json" + cache_dir = tmp_path / "cache" + + def mock_subprocess_run(cmd, capture_output, text): + class Result: + returncode = 0 + stdout = "" + stderr = "" + + if cmd[0] == "ssh": + assert ws.ssh.host in cmd + res = Result() + res.stdout = f"{remote_matched_path}\n" + return res + elif cmd[0] == "rsync": + dest_file = Path(cmd[-1]) + dest_file.parent.mkdir(parents=True, exist_ok=True) + dest_file.write_text( + '{"workspace": "/remote/workspace/path"}', encoding="utf-8" + ) + return Result() + return Result() + + monkeypatch.setattr("subprocess.run", mock_subprocess_run) + + result_path = fetch_remote_workspace_file(ws, target_job, cache_dir=cache_dir) + assert ( + result_path + == cache_dir / "jobs" / "my_task" / f"{target_job}12345" / "params.json" + ) + assert result_path.is_file() + + +def test_fetch_remote_workspace_file_custom_filename(tmp_path, monkeypatch): + """Verify fetching custom file (experimaestro.json) from remote workspace.""" + ws = make_mock_remote_workspace() + remote_matched_path = ( + f"{ws._raw_path}/jobs/my_task/ee76fb8f12345/experimaestro.json" + ) + cache_dir = tmp_path / "cache" + + def mock_subprocess_run(cmd, capture_output, text): + class Result: + returncode = 0 + stdout = "" + stderr = "" + + if cmd[0] == "ssh": + res = Result() + res.stdout = f"{remote_matched_path}\n" + return res + elif cmd[0] == "rsync": + dest_file = Path(cmd[-1]) + dest_file.parent.mkdir(parents=True, exist_ok=True) + dest_file.write_text("[]", encoding="utf-8") + return Result() + return Result() + + monkeypatch.setattr("subprocess.run", mock_subprocess_run) + + result_path = fetch_remote_workspace_file( + ws, "ee76fb8f", filename="experimaestro.json", cache_dir=cache_dir + ) + assert ( + result_path + == cache_dir / "jobs" / "my_task" / "ee76fb8f12345" / "experimaestro.json" + ) + assert result_path.is_file() + + +def test_fetch_remote_workspace_file_multiple_matches(tmp_path, monkeypatch, capsys): + """Verify warning printed and first match taken when multiple remote jobs match.""" + ws = make_mock_remote_workspace() + matches = [ + f"{ws._raw_path}/jobs/task_a/1111_job/params.json", + f"{ws._raw_path}/jobs/task_b/1112_job/params.json", + ] + cache_dir = tmp_path / "cache" + + def mock_subprocess_run(cmd, capture_output, text): + class Result: + returncode = 0 + stdout = "" + stderr = "" + + if cmd[0] == "ssh": + res = Result() + res.stdout = "\n".join(matches) + "\n" + return res + elif cmd[0] == "rsync": + dest_file = Path(cmd[-1]) + dest_file.parent.mkdir(parents=True, exist_ok=True) + dest_file.write_text("{}", encoding="utf-8") + return Result() + return Result() + + monkeypatch.setattr("subprocess.run", mock_subprocess_run) + + result_path = fetch_remote_workspace_file(ws, "111", cache_dir=cache_dir) + captured = capsys.readouterr() + assert "Warning: Multiple jobs (2) matched '111'" in captured.err + assert matches[0] in captured.err + assert result_path == cache_dir / "jobs" / "task_a" / "1111_job" / "params.json" + + +def test_fetch_remote_workspace_file_not_found(monkeypatch): + """Verify FileNotFoundError when no remote job matches.""" + ws = make_mock_remote_workspace() + + def mock_subprocess_run(cmd, capture_output, text): + class Result: + returncode = 0 + stdout = "" + stderr = "" + + return Result() + + monkeypatch.setattr("subprocess.run", mock_subprocess_run) + + with pytest.raises( + FileNotFoundError, + match="No job matching 'nonexistent' found in remote workspace 'mock-remote-ws'", + ): + fetch_remote_workspace_file(ws, "nonexistent") + + +def test_fetch_remote_workspace_file_ssh_error(monkeypatch): + """Verify RuntimeError when SSH command fails.""" + ws = make_mock_remote_workspace() + + def mock_subprocess_run(cmd, capture_output, text): + class Result: + returncode = 255 + stdout = "" + stderr = "Host unreachable" + + return Result() + + monkeypatch.setattr("subprocess.run", mock_subprocess_run) + + with pytest.raises( + RuntimeError, + match="Failed to query remote workspace 'mock-remote-ws' via SSH: Host unreachable", + ): + fetch_remote_workspace_file(ws, "some_job") + + +def test_fetch_remote_workspace_file_empty_target(): + """Verify FileNotFoundError when target is empty or root.""" + ws = make_mock_remote_workspace() + with pytest.raises(FileNotFoundError, match="Please specify a job ID or path"): + fetch_remote_workspace_file(ws, "") + + with pytest.raises(FileNotFoundError, match="Please specify a job ID or path"): + fetch_remote_workspace_file(ws, ".") + + +def test_find_params_file_remote_dispatch(tmp_path, monkeypatch): + """Verify that find_params_file dispatches to find_workspace_file for remote workspace.""" + ws = make_mock_remote_workspace() + expected_path = tmp_path / "params.json" + + monkeypatch.setattr( + "experimaestro.tools.inspect_params.discovery.resolve_workspace", lambda w: ws + ) + monkeypatch.setattr( + "experimaestro.tools.inspect_params.discovery.find_workspace_file", + lambda workspace, target, filename="params.json": expected_path, + ) + + result = find_params_file("my_remote_job", workspace="mock-remote-ws") + assert result == expected_path + + +def test_find_params_file_remote_workspace_overrides_absolute_path( + tmp_path, monkeypatch +): + """Verify that an absolute path is dispatched to remote workspace rather than local resolution.""" + ws = make_mock_remote_workspace() + expected_path = tmp_path / "params.json" + remote_abs_path = ( + f"{ws._raw_path}/jobs/my_task/b0ef27e326897e6d874df9a028ba1cbf09177958" + ) + + called_with = [] + monkeypatch.setattr( + "experimaestro.tools.inspect_params.discovery.resolve_workspace", lambda w: ws + ) + monkeypatch.setattr( + "experimaestro.tools.inspect_params.discovery.find_workspace_file", + lambda workspace, target, filename="params.json": ( + called_with.append((workspace, target, filename)) or expected_path + ), + ) + + result = find_params_file(remote_abs_path, workspace="mock-remote-ws") + assert result == expected_path + assert called_with == [(ws, remote_abs_path, "params.json")] + + +def test_fetch_remote_workspace_file_with_full_remote_absolute_path( + tmp_path, monkeypatch +): + """Verify fetch_remote_workspace_file correctly strips remote workspace prefix and /jobs/.""" + ws = make_mock_remote_workspace() + target_job = "b0ef27e3" + full_remote_path = ( + f"{ws._raw_path}/jobs/xpmir.evaluation.evaluate/{target_job}12345/params.json" + ) + cache_dir = tmp_path / "cache" + + executed_cmds = [] + + def mock_subprocess_run(cmd, capture_output, text): + executed_cmds.append(cmd) + + class Result: + returncode = 0 + stdout = "" + stderr = "" + + if cmd[0] == "ssh": + res = Result() + res.stdout = f"{full_remote_path}\n" + return res + elif cmd[0] == "rsync": + dest_file = Path(cmd[-1]) + dest_file.parent.mkdir(parents=True, exist_ok=True) + dest_file.write_text("{}", encoding="utf-8") + return Result() + return Result() + + monkeypatch.setattr("subprocess.run", mock_subprocess_run) + + result = fetch_remote_workspace_file(ws, full_remote_path, cache_dir=cache_dir) + assert ( + result + == cache_dir + / "jobs" + / "xpmir.evaluation.evaluate" + / f"{target_job}12345" + / "params.json" + ) + + # Check that SSH command queried the pattern properly without duplicated prefix + assert len(executed_cmds) == 2 + assert executed_cmds[0][0] == "ssh" + remote_py_cmd = executed_cmds[0][-1] + assert ( + f"{ws._raw_path}/jobs/xpmir.evaluation.evaluate/{target_job}12345*/params.json" + in remote_py_cmd + ) + + +def test_find_workspace_file_local(tmp_path): + """Verify find_workspace_file dispatches to local workspace directory.""" + job_dir = tmp_path / "jobs" / "task_x" / "9999" + job_dir.mkdir(parents=True) + f = job_dir / "experimaestro.json" + f.write_text("[]", encoding="utf-8") + + res = find_workspace_file(tmp_path, "9999", filename="experimaestro.json") + assert res == f.resolve() + + +def test_params_graph_list_payload(tmp_path): + """Verify that ParamsGraph supports direct list payloads (experimaestro.json format).""" + list_data = [ + { + "id": 1001, + "module": "mock_mod", + "type": "MockTokenizer", + "identifier": "tok123", + "fields": {"model_name": "bert"}, + }, + { + "id": 1002, + "module": "mock_mod", + "type": "MockModel", + "identifier": "mod123", + "task": 1002, + "fields": { + "tokenizer": {"type": "python", "value": 1001}, + "weights": {"type": "path.serialized", "value": "model.safetensors"}, + }, + }, + ] + + p = tmp_path / "experimaestro.json" + p.write_text(json.dumps(list_data), encoding="utf-8") + (tmp_path / "model.safetensors").write_text("weights") + + graph = ParamsGraph.from_file(p) + assert len(graph.objects) == 2 + assert graph.workspace_path is None + assert graph.version == 0 + assert graph.experimaestro_version == "unknown" + assert graph.root_task_id == 1002 + + model_node = graph.objects[1002] + assert model_node.is_root_task is True + assert ("tokenizer", 1001) in model_node.outgoing_refs + + # Path resolution for path.serialized relative to job_dir + weights_path = graph.resolve_path(model_node.fields["weights"]) + assert weights_path is not None + assert weights_path.exists is True + assert weights_path.resolved == tmp_path / "model.safetensors" + + # Python snippet formatting + py_code = graph.to_python_snippet(1002) + assert "MockModel.C(" in py_code + assert 'Path("model.safetensors")' in py_code + + +def test_find_params_file_directory_with_experimaestro_json(tmp_path): + """Verify finding experimaestro.json in directory when params.json is absent.""" + exp_file = tmp_path / "experimaestro.json" + exp_file.write_text("[]", encoding="utf-8") + + res = find_params_file(str(tmp_path)) + assert res == exp_file + + +@pytest.mark.anyio +async def test_experimaestro_json_tui_app(tmp_path): + """Verify that ParamsInspectorApp mounts and renders multi-task roots from list payloads.""" + list_data = [ + { + "id": 2001, + "module": "pkg", + "type": "DatasetTask", + "identifier": "ds2001", + "task": 2001, + "fields": {"size": 100}, + }, + { + "id": 2002, + "module": "pkg", + "type": "EvalTask", + "identifier": "eval2002", + "task": 2002, + "fields": {"metric": "ndcg"}, + }, + ] + p = tmp_path / "experimaestro.json" + p.write_text(json.dumps(list_data), encoding="utf-8") + + app = ParamsInspectorApp(p) + async with app.run_test(): + tree = app.query_one(ParamsTree) + assert len(tree.root.children) >= 2 + labels = [str(c.label) for c in tree.root.children] + assert any("Root Task" in lbl for lbl in labels) + assert any("Other Tasks" in lbl for lbl in labels) + + +@pytest.mark.anyio +async def test_root_model_primary_placement(tmp_path): + """Verify that core fields ('value', 'model') render as primary at the root rather than in listeners.""" + # Construct a task where 'settings' alphabetically precedes 'value', + # and 'settings' contains a nested listener referencing the same model. + data = [ + { + "id": 1, + "module": "mock", + "type": "TargetModel", + "identifier": "model123", + "fields": {"hidden_size": 768}, + }, + { + "id": 2, + "module": "mock", + "type": "Listener", + "identifier": "listener123", + "fields": { + "scorer": {"type": "python", "value": 1}, + }, + }, + { + "id": 3, + "module": "mock", + "type": "RootTask", + "identifier": "roottask123", + "task": 3, + "fields": { + "settings": {"type": "python", "value": 2}, + "value": {"type": "python", "value": 1}, + }, + }, + ] + p = tmp_path / "experimaestro.json" + p.write_text(json.dumps(data), encoding="utf-8") + + app = ParamsInspectorApp(p) + async with app.run_test(): + tree = app.query_one(ParamsTree) + # Root task is child 1 (after metadata) + root_node = tree.root.children[1] + + # The primary node for model #1 must be a direct child of root task under 'value' + primary_model = tree.primary_nodes.get(1) + assert primary_model is not None + assert primary_model.parent == root_node + assert "value:" in str(primary_model.label) + assert "Shared" not in str(primary_model.label) + + # The listener's reference under 'settings' must be marked as Shared + settings_node = [c for c in root_node.children if "settings:" in str(c.label)][ + 0 + ] + listener_scorer = settings_node.children[0] + assert "Shared" in str(listener_scorer.label) + assert listener_scorer.data.target_object_id == 1 + + +def test_cli_inspect_help(): + """Verify experimaestro inspect --help CLI command.""" + from click.testing import CliRunner + from experimaestro.cli import cli + + runner = CliRunner() + res = runner.invoke(cli, ["inspect", "--help"]) + assert res.exit_code == 0 + assert "Interactive TUI Parameters Inspector" in res.output + assert "--workspace" in res.output diff --git a/src/experimaestro/tests/test_xpm_dag.py b/src/experimaestro/tests/test_xpm_dag.py index b9caa196..493d27a2 100644 --- a/src/experimaestro/tests/test_xpm_dag.py +++ b/src/experimaestro/tests/test_xpm_dag.py @@ -407,3 +407,154 @@ def test_matplotlib_lazy_import(tmp_path, monkeypatch): assert res.exit_code == 0 assert "matplotlib" not in sys.modules assert "matplotlib.pyplot" not in sys.modules + + +def test_draw_dag_depths_alignment_and_deep_deps(): + """Verify that compute_dag_depths aligns same-stage tasks and pushes deeper dependencies.""" + import networkx as nx + from experimaestro.tools.xpm_dag.draw_dag import compute_dag_depths + + def make_node(name: str): + return type(name, (), {})() + + full_jobs = { + "dataset": make_node("MNISTLabelledImages"), + "extra_prep": make_node("PreprocessTask"), + "deep_step": make_node("DeepHelperTask"), + "learn_shallow": make_node("Learn"), + "learn_deep": make_node("Learn"), + "eval_shallow": make_node("Evaluate"), + "eval_deep": make_node("Evaluate"), + } + + dag = nx.DiGraph() + for jid in full_jobs: + dag.add_node(jid) + + # Shallow pipeline: dataset -> learn_shallow -> eval_shallow + dag.add_edge("dataset", "learn_shallow") + dag.add_edge("learn_shallow", "eval_shallow") + + # Deep pipeline: dataset -> extra_prep -> deep_step -> learn_deep -> eval_deep + dag.add_edge("dataset", "extra_prep") + dag.add_edge("extra_prep", "deep_step") + dag.add_edge("deep_step", "learn_deep") + dag.add_edge("learn_deep", "eval_deep") + + depths = compute_dag_depths( + dag, + full_jobs, + align_types=True, + aligned_class_names={"Learn", "Evaluate"}, + ) + + assert depths["learn_shallow"] == depths["learn_deep"] == 3 + assert depths["eval_shallow"] >= 4 + assert depths["eval_deep"] >= 4 + + for u, v in dag.edges(): + assert depths[v] > depths[u], f"Edge {u} -> {v} violated monotonicity" + + +def test_draw_dag_filter_helpers(tmp_path): + """Verify tag parsing, config listing, and subgraph extraction.""" + from experimaestro.tools.xpm_dag.draw_dag import ( + filter_subgraph_by_config, + parse_filter_arg, + ) + + # 1. Test parse_filter_arg + parsed = parse_filter_arg("lr=0.001,kernel_size=3,name=cnn,flag=true") + assert parsed == {"lr": 0.001, "kernel_size": 3, "name": "cnn", "flag": "true"} + + # 2. Test filter_subgraph_by_config + class MockObj: + pass + + full_jobs = {"root": MockObj(), "j1": MockObj(), "j2": MockObj(), "j3": MockObj()} + dependencies = { + "j1": ["root"], + "j2": ["root", "j1"], + "j3": ["root"], + } + job_tags_dict = { + "j1": {"lr": 0.01, "layers": 2}, + "j2": {"lr": 0.01, "layers": 2}, + "j3": {"lr": 0.001, "layers": 3}, + } + + filtered_jobs, filtered_deps = filter_subgraph_by_config( + full_jobs, + dependencies, + job_tags_dict, + target_tags={"lr": 0.01, "layers": 2}, + ) + + assert set(filtered_jobs.keys()) == {"root", "j1", "j2"} + assert "j3" not in filtered_jobs + assert filtered_deps["j1"] == ["root"] + assert filtered_deps["j2"] == ["root", "j1"] + + +def test_resolve_run_dir_error_reporting(tmp_path, caplog): + """Verify that resolve_run_dir provides helpful error messages listing available experiments and runs.""" + from experimaestro.tools.xpm_dag.xpm_dag import resolve_run_dir + + ws_dir = tmp_path / "test_ws" + exp_dir = ws_dir / "experiments" + (exp_dir / "exp_alpha" / "20260101_120000").mkdir(parents=True) + (exp_dir / "exp_beta" / "20260102_150000").mkdir(parents=True) + + with caplog.at_level(logging.ERROR): + # 1. Non-existent experiment + res = resolve_run_dir(str(ws_dir), "exp_gamma", None) + assert res is None + assert "Experiment 'exp_gamma' not found" in caplog.text + assert "'exp_alpha'" in caplog.text and "'exp_beta'" in caplog.text + + caplog.clear() + + # 2. Non-existent run + res = resolve_run_dir(str(ws_dir), "exp_alpha", "wrong_run_id") + assert res is None + assert "Run 'wrong_run_id' not found" in caplog.text + assert "'20260101_120000'" in caplog.text + + +def test_resolve_run_dir_remote_ssh(monkeypatch, tmp_path): + """Verify that resolve_run_dir delegates to remote SSH fetcher when workspace is remote.""" + from pathlib import Path + from experimaestro.settings import SSHSettings, WorkspaceSettings + from experimaestro.tools.xpm_dag.xpm_dag import resolve_run_dir + + ssh = SSHSettings(host="cluster.example.org") + ws = WorkspaceSettings( + id="cluster_ws", + path=Path("/remote/workspace"), + ssh=ssh, + ) + + fake_run_dir = tmp_path / "cached_run" + fake_run_dir.mkdir() + + called = [] + + def mock_fetch(remote_ws, exp_id, r_id): + called.append((remote_ws.id, exp_id, r_id)) + return "20260101_120000", fake_run_dir + + monkeypatch.setattr( + "experimaestro.tools.xpm_dag.xpm_dag.find_workspace", + lambda **kwargs: ws, + ) + monkeypatch.setattr( + "experimaestro.utils.remote.find_remote_workspace_experiment_run", + mock_fetch, + ) + + resolved_id, run_d = resolve_run_dir( + "cluster_ws", "my_remote_exp", "20260101_120000" + ) + assert resolved_id == "20260101_120000" + assert run_d == fake_run_dir + assert called == [("cluster_ws", "my_remote_exp", "20260101_120000")] diff --git a/src/experimaestro/tools/inspect_params/__init__.py b/src/experimaestro/tools/inspect_params/__init__.py new file mode 100644 index 00000000..8f11443b --- /dev/null +++ b/src/experimaestro/tools/inspect_params/__init__.py @@ -0,0 +1,69 @@ +"""Experimaestro Parameters Inspector tool. + +Inspects params.json and experimaestro.json files with a rich, collapsible +configuration tree, cycle detection, path resolution, and Python code synthesis. +""" + +from __future__ import annotations + +from typing import Optional + +from experimaestro.tools.inspect_params.app import ( + NodeData, + NodeDetailPane, + NodeKind, + ParamsInspectorApp, + ParamsTree, +) +from experimaestro.tools.inspect_params.discovery import ( + find_local_workspace_file, + find_params_file, + find_workspace_file, + resolve_workspace, +) +from experimaestro.tools.inspect_params.models import ( + ObjectNode, + ParamsGraph, + ResolvedPath, +) +from experimaestro.utils.remote import ( + clean_job_target, + fetch_remote_file_rsync, + fetch_remote_workspace_file, + find_remote_workspace_file, + get_session_cache_dir, + resolve_job_file_pattern, + run_ssh_command, +) + +__all__ = [ + "NodeData", + "NodeDetailPane", + "NodeKind", + "ObjectNode", + "ParamsGraph", + "ParamsInspectorApp", + "ParamsTree", + "ResolvedPath", + "clean_job_target", + "fetch_remote_file_rsync", + "fetch_remote_workspace_file", + "find_local_workspace_file", + "find_params_file", + "find_remote_workspace_file", + "find_workspace_file", + "get_session_cache_dir", + "resolve_job_file_pattern", + "resolve_workspace", + "run_inspector", + "run_ssh_command", +] + + +def run_inspector( + target: Optional[str] = None, workspace: Optional[str] = None +) -> None: + """Find parameters file and launch the interactive Textual inspector app.""" + params_path = find_params_file(target, workspace) + app = ParamsInspectorApp(params_path) + app.run() diff --git a/src/experimaestro/tools/inspect_params/app.py b/src/experimaestro/tools/inspect_params/app.py new file mode 100644 index 00000000..b62f0cc3 --- /dev/null +++ b/src/experimaestro/tools/inspect_params/app.py @@ -0,0 +1,1063 @@ +# ruff: noqa: T201 +"""Interactive Textual TUI widgets and application for Experimaestro parameters inspector.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import json +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +import pyperclip # type: ignore[import-untyped] +from rich.console import Group +from rich.panel import Panel +from rich.syntax import Syntax +from rich.table import Table +from rich.text import Text + +from textual.app import App, ComposeResult +from textual.binding import Binding +from textual.containers import Horizontal, Vertical, VerticalScroll +from textual.widgets import Footer, Header, Input, Static, Tree +from textual.widgets.tree import TreeNode + +from experimaestro.tools.inspect_params.models import ( + ParamsGraph, + ResolvedPath, +) + +logger = logging.getLogger("experimaestro") + + +class NodeKind(Enum): + """Enumeration of semantic node classifications in the parameters tree.""" + + ROOT_TASK = "root" + OBJECT = "obj" + CATEGORY = "cat" + FIELD_PRIMITIVE = "prim" + FIELD_PATH = "path" + FIELD_COLLECTION = "coll" + METADATA = "meta" + CARBON = "carbon" + CYCLE = "cycle" + SHARED_REF = "shared" + + +@dataclass +class NodeData: + """Payload metadata attached to a Textual Tree node. + + Attributes: + kind: Semantic classification of the tree item. + title: Display title or name. + object_id: ID of the underlying Experimaestro object, if applicable. + field_name: Parameter field name if this node represents a field. + field_value: Raw or resolved value of the parameter. + resolved_path: ResolvedPath instance if this node represents a path. + target_object_id: Primary object ID if this node is a shared reference or jump target. + cycle_chain: Sequence of object IDs forming a circular reference chain. + is_shared: Whether this object is referenced multiple times in the DAG. + """ + + kind: NodeKind + title: str + object_id: Optional[int] = None + field_name: Optional[str] = None + field_value: Any = None + resolved_path: Optional[ResolvedPath] = None + target_object_id: Optional[int] = None + cycle_chain: Optional[List[int]] = None + is_shared: bool = False + + +class ParamsTree(Tree[NodeData]): + """Hierarchical Task Configuration Tree widget. + + Renders Experimaestro tasks, configs, hyperparameters, paths, and collections + in an interactive collapsible tree structure with cycle detection and shared + DAG reference indication. + """ + + def __init__(self, graph: ParamsGraph) -> None: + """Initialize the parameters tree widget. + + Args: + graph: The indexed ParamsGraph instance to display. + """ + super().__init__("Parameters") + self.graph = graph + self.group_fields: bool = True + self.filter_query: str = "" + self.show_absolute_paths: bool = False + self.primary_nodes: Dict[int, TreeNode[NodeData]] = {} + self.matched_nodes: List[TreeNode[NodeData]] = [] + self.match_count: int = 0 + + def on_mount(self) -> None: + """Configure tree display settings and build initial tree structure.""" + self.show_root = False + self.guide_depth = 3 + self.rebuild() + + def set_filter(self, q: str) -> None: + """Filter the tree view by search query string. + + Args: + q: Query string to filter fields or objects. + """ + self.filter_query = q.strip().lower() + self.rebuild() + + def toggle_grouping(self) -> None: + """Toggle between categorized field groups and alphabetical sorting.""" + self.group_fields = not self.group_fields + self.rebuild() + + def toggle_paths(self) -> None: + """Toggle between absolute and relative path display.""" + self.show_absolute_paths = not self.show_absolute_paths + self.rebuild() + + def _matches(self, *texts: Any) -> bool: + """Check if any of the provided strings contain the active search query.""" + if not self.filter_query: + return False + q = self.filter_query + return any(isinstance(t, str) and q in t.lower() for t in texts) + + def _apply_highlight(self, text: Text) -> None: + """Highlight active search query terms in a Rich Text instance.""" + if self.filter_query: + text.highlight_words( + [self.filter_query], + style="bold black on #f9e2af", + case_sensitive=False, + ) + + def rebuild(self) -> None: + """Clear and rebuild the entire tree from the underlying graph.""" + self.clear() + self.primary_nodes.clear() + self.matched_nodes.clear() + self.match_count = 0 + + # Metadata header + meta = self.root.add( + Text("ℹ️ Metadata & Environment", style="bold cyan"), + data=NodeData(NodeKind.METADATA, "Metadata"), + ) + meta.add_leaf( + Text(f"Workspace: {self.graph.workspace_path or 'unknown'}", style="dim"), + data=NodeData(NodeKind.METADATA, "Workspace"), + ) + meta.add_leaf( + Text( + f"v{self.graph.version} · experimaestro {self.graph.experimaestro_version}", + style="dim", + ), + data=NodeData(NodeKind.METADATA, "Version"), + ) + if self.graph.carbon.get("enabled"): + meta.add_leaf( + Text( + f"🌱 Carbon: {self.graph.carbon.get('provider')} (ON)", + style="green", + ), + data=NodeData(NodeKind.CARBON, "Carbon", field_value=self.graph.carbon), + ) + + # Root task + seen: Set[int] = set() + rid = self.graph.root_task_id + if rid and rid in self.graph.objects: + self._render_obj( + self.root, rid, ancestors=(), seen=seen, label="📦 Root Task" + ) + else: + self.root.add_leaf("[dim]No root task found[/dim]") + + # Other root tasks / components if present (e.g. in multi-task graphs / experimaestro.json) + other_roots = [ + oid + for oid, node in self.graph.objects.items() + if oid not in seen and not node.incoming_refs + ] + if other_roots: + other_container = self.root.add( + Text( + f"📋 Other Tasks & Configurations ({len(other_roots)})", + style="bold cyan", + ), + data=NodeData(NodeKind.CATEGORY, "Other Tasks"), + ) + for roid in other_roots: + rnode = self.graph.objects[roid] + lbl = "⚡ Task" if rnode.is_task else "🧩 Config" + self._render_obj( + other_container, roid, ancestors=(), seen=seen, label=lbl + ) + + # Fallback for disconnected cycles or orphaned nodes + orphans = [oid for oid in self.graph.objects if oid not in seen] + if orphans: + orphan_container = self.root.add( + Text( + f"🧩 Additional Objects ({len(orphans)})", + style="bold yellow", + ), + data=NodeData(NodeKind.CATEGORY, "Additional Objects"), + ) + for roid in orphans: + if roid not in seen: + self._render_obj( + orphan_container, roid, ancestors=(), seen=seen, label="Object" + ) + + if self.filter_query: + if self.match_count == 0: + self.root.add_leaf( + Text( + f"❌ No matching parameters or configs for '{self.filter_query}'", + style="dim red", + ), + data=NodeData(NodeKind.METADATA, "No Match"), + ) + else: + for n in self.matched_nodes: + if n.children: + n.expand() + curr = n.parent + while curr is not None: + curr.expand() + curr = curr.parent + self.move_cursor(self.matched_nodes[0]) + self.scroll_to_node(self.matched_nodes[0]) + else: + for c in self.root.children: + c.expand() + + @staticmethod + def _order_fields(fields: Dict[str, Any]) -> List[Tuple[str, Any]]: + """Order fields prioritizing core payload components over auxiliary metadata.""" + CORE_KEYS = { + "model": 0, + "value": 0, + "task": 1, + "module": 1, + "dataset": 2, + "data": 2, + } + AUX_KEYS = { + "settings": 10, + "hooks": 11, + "listeners": 12, + "callbacks": 13, + } + return sorted( + fields.items(), + key=lambda kv: ( + CORE_KEYS.get(kv[0], 5), + AUX_KEYS.get(kv[0], 5), + kv[0], + ), + ) + + def _render_obj( + self, + parent: TreeNode[NodeData], + oid: int, + ancestors: Tuple[int, ...], + seen: Set[int], + label: str = "", + ) -> TreeNode[NodeData]: + """Render an ObjectNode into the tree. + + Handles cycle detection, shared DAG references, field categorization, + and recursive rendering of child components and init tasks. + + Args: + parent: Parent tree node to attach to. + oid: Numeric ID of the object to render. + ancestors: Tuple of ancestor object IDs for cycle detection. + seen: Set of already rendered object IDs in the tree. + label: Optional field or role label prefix. + + Returns: + The constructed TreeNode. + """ + node = self.graph.objects[oid] + hid = node.identifier[:8] if node.identifier else "no-id" + + # Loop / Cycle guard + if oid in ancestors: + txt = Text() + if label: + txt.append(f"{label}: ", style="bold white") + txt.append("🔁 [CYCLE] ", style="bold red") + txt.append(f"{node.type_name} [#{oid} · {hid}]", style="dim red") + return parent.add_leaf( + txt, + data=NodeData( + NodeKind.CYCLE, + node.type_name, + object_id=oid, + cycle_chain=list(ancestors) + [oid], + ), + ) + + # Shared DAG reference + is_shared = len(node.incoming_refs) > 1 + if oid in seen: + txt = Text() + if label: + txt.append(f"{label}: ", style="bold white") + txt.append("↪ [Shared] ", style="bold yellow") + txt.append(f"{node.type_name} [#{oid} · {hid}]", style="dim yellow") + tn = parent.add( + txt, + data=NodeData( + NodeKind.SHARED_REF, + node.type_name, + object_id=oid, + target_object_id=oid, + is_shared=True, + ), + ) + tn.add_leaf( + Text("(Press 'j' to jump to primary)", style="dim italic"), + data=NodeData(NodeKind.SHARED_REF, "Hint", target_object_id=oid), + ) + return tn + + seen.add(oid) + icon = "📦" if node.is_root_task else ("⚡" if node.is_task else "🧩") + st = ( + "bold magenta" + if node.is_root_task + else ("bold cyan" if node.is_task else "bold blue") + ) + txt = Text() + if label: + txt.append(f"{label}: ", style="bold white") + txt.append(f"{icon} {node.type_name} ", style=st) + txt.append(f"[{hid}] ", style="dim") + if is_shared: + txt.append(f"(shared x{len(node.incoming_refs)})", style="italic yellow") + + is_match = self._matches( + node.type_name, node.qualified_name, node.identifier, label + ) + if is_match: + self._apply_highlight(txt) + + tree_node = parent.add( + txt, + data=NodeData( + NodeKind.ROOT_TASK if node.is_root_task else NodeKind.OBJECT, + node.qualified_name, + object_id=oid, + is_shared=is_shared, + ), + ) + self.primary_nodes[oid] = tree_node + if is_match: + self.matched_nodes.append(tree_node) + self.match_count += 1 + next_anc = ancestors + (oid,) + + # Categorize fields + sub_c, paths, hparams, colls = [], [], [], [] + for fn, fv in self._order_fields(node.fields): + if isinstance(fv, dict) and fv.get("type") == "python": + sub_c.append((fn, fv)) + elif isinstance(fv, dict) and fv.get("type") in ("path", "path.serialized"): + paths.append((fn, fv)) + elif isinstance(fv, (list, dict)) and not ( + isinstance(fv, dict) and fv.get("type") == "enum" + ): + colls.append((fn, fv)) + else: + hparams.append((fn, fv)) + + def _add_inits(target: TreeNode[NodeData]) -> None: + if node.init_tasks: + itn = target.add( + Text( + f"⚡ Init Tasks ({len(node.init_tasks)})", style="bold magenta" + ), + data=NodeData(NodeKind.CATEGORY, "Init Tasks"), + ) + for i, it in enumerate(node.init_tasks): + if it in self.graph.objects: + self._render_obj(itn, it, next_anc, seen, label=f"[{i}]") + itn.expand() + + if self.group_fields and (len(node.fields) > 3 or node.init_tasks): + for gtitle, gstyle, items in [ + ("⚙️ Hyperparameters", "bold yellow", hparams), + ("📁 Paths & Outputs", "bold blue", paths), + ("🧩 Sub-Configurations", "bold cyan", sub_c), + ]: + if items: + gn = tree_node.add( + Text(gtitle, style=gstyle), + data=NodeData(NodeKind.CATEGORY, gtitle), + ) + for fn, fv in items: + self._render_val(gn, fn, fv, next_anc, seen) + gn.expand() + + for fn, fv in colls: + self._render_val(tree_node, fn, fv, next_anc, seen) + _add_inits(tree_node) + else: + for fn, fv in self._order_fields(node.fields): + self._render_val(tree_node, fn, fv, next_anc, seen) + _add_inits(tree_node) + + return tree_node + + def _render_val( + self, + parent: TreeNode[NodeData], + fn: str, + v: Any, + anc: Tuple[int, ...], + seen: Set[int], + ) -> None: + """Render an individual field value (sub-object, path, collection, or primitive). + + Args: + parent: Parent tree node to attach to. + fn: Field name. + v: Field value. + anc: Tuple of ancestor object IDs. + seen: Set of visited object IDs. + """ + if ( + isinstance(v, dict) + and v.get("type") == "python" + and "value" in v + and int(v["value"]) in self.graph.objects + ): + self._render_obj(parent, int(v["value"]), anc, seen, label=fn) + return + + rp = self.graph.resolve_path(v) + if rp: + is_match = self._matches(fn, str(rp.resolved), rp.raw_value) + txt = Text(f"{fn}: ", style="bold white") + pstr = ( + str(rp.resolved) + if self.show_absolute_paths + else f"{rp.raw_value} [{rp.base}]" + ) + sz = ( + f" ({self._fmt_sz(rp.size_bytes)})" + if rp.exists and rp.size_bytes is not None + else "" + ) + txt.append( + f"{'📁' if rp.is_dir else '📄'} {pstr}{sz} ", style="underline blue" + ) + if not rp.exists: + txt.append("[not found]", style="red") + if is_match: + self._apply_highlight(txt) + + leaf_node = parent.add_leaf( + txt, + data=NodeData( + NodeKind.FIELD_PATH, + fn, + field_name=fn, + field_value=v, + resolved_path=rp, + ), + ) + if is_match: + self.matched_nodes.append(leaf_node) + self.match_count += 1 + return + + if isinstance(v, (list, dict)) and not ( + isinstance(v, dict) and v.get("type") == "enum" + ): + is_match = self._matches(fn) + if isinstance(v, list): + desc = f"[{len(v)} items]" + txt = Text(f"{fn}: ", style="bold white") + Text(desc, style="cyan") + if is_match: + self._apply_highlight(txt) + cn = parent.add( + txt, + data=NodeData( + NodeKind.FIELD_COLLECTION, fn, field_name=fn, field_value=v + ), + ) + if is_match: + self.matched_nodes.append(cn) + self.match_count += 1 + for k_idx, val in enumerate(v): + self._render_val(cn, f"[{k_idx}]", val, anc, seen) + else: + desc = f"{{{len(v)} entries}}" + txt = Text(f"{fn}: ", style="bold white") + Text(desc, style="cyan") + if is_match: + self._apply_highlight(txt) + cn = parent.add( + txt, + data=NodeData( + NodeKind.FIELD_COLLECTION, fn, field_name=fn, field_value=v + ), + ) + if is_match: + self.matched_nodes.append(cn) + self.match_count += 1 + for k_key, val in sorted(v.items()): + self._render_val(cn, str(k_key), val, anc, seen) + return + + is_match = self._matches(fn, str(v)) + txt = Text(f"{fn}: ", style="bold white") + if isinstance(v, dict) and v.get("type") == "enum": + txt.append(f"{v.get('enum')}.{v.get('value')}", style="bold yellow") + elif isinstance(v, str): + txt.append(f'"{v}"', style="green") + elif isinstance(v, bool): + txt.append(str(v), style="bold yellow") + elif isinstance(v, (int, float)): + txt.append(str(v), style="magenta") + elif v is None: + txt.append("null", style="dim") + else: + txt.append(str(v), style="white") + + if is_match: + self._apply_highlight(txt) + + leaf_node = parent.add_leaf( + txt, + data=NodeData(NodeKind.FIELD_PRIMITIVE, fn, field_name=fn, field_value=v), + ) + if is_match: + self.matched_nodes.append(leaf_node) + self.match_count += 1 + + @staticmethod + def _fmt_sz(sz: int) -> str: + """Format file size in human-readable B, KB, or MB units. + + Args: + sz: Size in bytes. + + Returns: + Human-readable size string. + """ + if sz < 1024: + return f"{sz} B" + if sz < 1024 * 1024: + return f"{sz / 1024:.1f} KB" + return f"{sz / (1024 * 1024):.1f} MB" + + +class NodeDetailPane(VerticalScroll): + """Right-hand detail inspector or raw JSON viewer widget. + + Displays structured cards for objects, paths, primitives, carbon metrics, + and metadata, or syntax-highlighted raw JSON. + """ + + def __init__(self, graph: ParamsGraph) -> None: + """Initialize the detail inspector pane. + + Args: + graph: The ParamsGraph instance being inspected. + """ + super().__init__(id="detail-pane") + self.graph = graph + self.current_data: Optional[NodeData] = None + self.show_raw_json: bool = False + + def compose(self) -> ComposeResult: + """Compose child widgets for the detail pane.""" + yield Static(id="detail-content") + + def toggle_raw_json(self) -> None: + """Toggle between formatted inspector card view and raw JSON view.""" + self.show_raw_json = not self.show_raw_json + self.update_detail(self.current_data) + + def update_detail(self, data: Optional[NodeData]) -> None: + """Update displayed card or raw JSON based on selected tree node data. + + Args: + data: NodeData of the selected tree item, or None. + """ + self.current_data = data + w = self.query_one("#detail-content", Static) + if not data: + w.update(Panel("No node selected", title="Detail Inspector")) + return + + if self.show_raw_json: + target = ( + self.graph.objects[data.object_id].fields + if data.object_id and data.object_id in self.graph.objects + else {data.field_name: data.field_value} + if data.field_name + else self.graph.raw_data + ) + w.update( + Panel( + Syntax( + json.dumps(target, indent=2), + "json", + theme="monokai", + word_wrap=True, + ), + title="Raw JSON ('J' toggle)", + ) + ) + return + + k = data.kind + if k in ( + NodeKind.OBJECT, + NodeKind.ROOT_TASK, + NodeKind.SHARED_REF, + NodeKind.CYCLE, + ): + self._render_obj_card(w, data) + elif k == NodeKind.FIELD_PATH and data.resolved_path: + self._render_path_card(w, data) + elif k == NodeKind.FIELD_PRIMITIVE: + rows = [ + ("Field", str(data.field_name)), + ( + "Type", + type(data.field_value).__name__ + if data.field_value is not None + else "null", + ), + ("Value", f"[bold yellow]{repr(data.field_value)}[/bold yellow] ('c')"), + ] + w.update(self._make_table(rows)) + elif k == NodeKind.CARBON: + c = data.field_value or {} + rows = [ + ( + "Tracking", + "[green]ENABLED[/green]" if c.get("enabled") else "Disabled", + ), + ("Provider", str(c.get("provider"))), + ("Region", str(c.get("region") or "Auto")), + ("Interval", f"{c.get('report_interval_s')}s"), + ] + w.update(self._make_table(rows, "🌱 Carbon Tracking")) + elif k == NodeKind.METADATA: + rows = [ + ("Workspace", str(self.graph.workspace_path or "unknown")), + ("Job Dir", str(self.graph.job_path or "unknown")), + ("Schema", f"v{self.graph.version}"), + ("Experimaestro", self.graph.experimaestro_version), + ("Objects", str(len(self.graph.objects))), + ] + w.update(self._make_table(rows, "ℹ️ Environment")) + else: + w.update(Panel(repr(data.field_value), title=data.title)) + + def _render_obj_card(self, w: Static, d: NodeData) -> None: + """Render detailed object card with properties, Python code, and references. + + Args: + w: Target Static widget. + d: Selected tree node data. + """ + oid = d.object_id or d.target_object_id + node = self.graph.objects.get(oid) if oid else None + if not node: + w.update(Panel("Unknown Object", style="red")) + return + + role = ( + "Root Task 📦" + if node.is_root_task + else ("Init Task ⚡" if node.is_task else "Config 🧩") + ) + refs = ( + ", ".join(f"#{c}.{f}" for c, f in node.incoming_refs) + if node.incoming_refs + else "None (Root / Unreferenced)" + ) + rows = [ + ("Type", f"[bold white]{node.type_name}[/bold white]"), + ("Full Name", f"[cyan]{node.qualified_name}[/cyan]"), + ("Object ID", f"#{node.id}"), + ("Role", role), + ("Identifier", f"[bold yellow]{node.identifier}[/bold yellow] ('c')"), + ("Fields", f"{len(node.fields)} fields"), + ] + parts: List[Any] = [self._make_table(rows)] + + if d.kind == NodeKind.CYCLE: + parts.insert( + 0, + Panel( + f"[bold red]Circular Reference![/bold red]\nChain: {' ➔ '.join(f'#{c}' for c in (d.cycle_chain or []))}", + style="red", + title="⚠️ Loop Guard", + ), + ) + if node.docstring: + parts.append(Panel(node.docstring, title="Docstring", style="dim italic")) + + code = self.graph.to_python_snippet(node.id) + parts.append( + Panel( + Syntax(code, "python", theme="monokai", word_wrap=True), + title="Python .C(...) ('y' copy)", + subtitle="Press 'J' for Raw JSON", + ) + ) + parts.append(self._make_table([("Referenced by", refs)])) + w.update(Group(*parts)) + + def _render_path_card(self, w: Static, d: NodeData) -> None: + """Render detailed path card with filesystem metadata and preview. + + Args: + w: Target Static widget. + d: Selected tree node data. + """ + rp = d.resolved_path + if not rp: + return + st = "[green]✓ Exists[/green]" if rp.exists else "[red]✗ Not Found[/red]" + rows = [ + ("Field", f"[bold white]{d.field_name}[/bold white]"), + ("Base", rp.base), + ("Raw Value", rp.raw_value), + ("Resolved", f"[cyan]{rp.resolved}[/cyan] ('c')"), + ("Status", st), + ] + if rp.exists: + rows.append(("Kind", "Directory 📁" if rp.is_dir else "File 📄")) + if rp.size_bytes is not None: + rows.append(("Size", ParamsTree._fmt_sz(rp.size_bytes))) + if rp.mtime: + rows.append(("Modified", rp.mtime.strftime("%Y-%m-%d %H:%M:%S"))) + + parts: List[Any] = [self._make_table(rows)] + if rp.exists and not rp.is_dir and (rp.size_bytes or 0) < 500_000: + try: + txt = "\n".join( + rp.resolved.read_text( + encoding="utf-8", errors="replace" + ).splitlines()[:12] + ) + parts.append( + Panel(txt, title=f"Preview: {rp.resolved.name}", style="dim") + ) + except Exception: + pass + w.update(Group(*parts)) + + @staticmethod + def _make_table(rows: List[Tuple[str, str]], title: Optional[str] = None) -> Any: + """Construct a formatted two-column table or panel. + + Args: + rows: List of (key, value) string pairs. + title: Optional panel title. + + Returns: + Rich Table or Panel instance. + """ + tbl = Table(show_header=False, box=None, padding=(0, 1)) + tbl.add_column("Key", style="bold cyan", width=14) + tbl.add_column("Value") + for k, v in rows: + tbl.add_row(k, v) + return Panel(tbl, title=title) if title else tbl + + +# --------------------------------------------------------------------------- +# Standalone App +# --------------------------------------------------------------------------- + + +class ParamsInspectorApp(App): + """Standalone Textual App for inspecting experimaestro params.json. + + Features interactive tree navigation, categorization, path resolution, + fuzzy search filtering, Python snippet copying, and raw JSON inspection. + """ + + TITLE = "Experimaestro Parameters Inspector" + CSS = """ + Screen { background: #1e1e2e; color: #cdd6f4; } + #mode-bar { height: auto; background: #181825; padding: 0 1; border-bottom: solid #313244; } + .title-badge { padding: 0 1; background: #89b4fa; color: #11111b; text-style: bold; } + #mode-hints { color: #a6adc8; padding-left: 2; } + #search-bar-container { height: auto; background: #313244; padding: 0 1; align: left middle; } + #search-bar-container.hidden { display: none; } + #search-input { width: 1fr; } + #search-input.not-found { color: #f38ba8; border: tall #f38ba8; } + #search-status { padding: 0 1; width: auto; height: 100%; content-align: center middle; } + #search-status.not-found { color: #f38ba8; text-style: bold; } + #search-status.matched { color: #a6e3a1; text-style: bold; } + #split-container { width: 100%; height: 1fr; } + #tree-container { width: 60%; height: 100%; border-right: solid #313244; } + ParamsTree { width: 100%; height: 100%; background: #1e1e2e; } + #detail-pane { width: 40%; height: 100%; padding: 1; background: #181825; } + #detail-content { width: 100%; height: auto; } + """ + + BINDINGS = [ + Binding("g", "toggle_grouping", "Group Fields", show=True), + Binding("p", "toggle_paths", "Paths", show=True), + Binding("e", "expand_all", "Expand All", show=True), + Binding("E", "collapse_all", "Collapse", show=True), + Binding("slash", "toggle_search", "Search", key_display="/", show=True), + Binding("c", "copy_value", "Copy", show=True), + Binding("y", "copy_python", "Copy Python", show=True), + Binding("J", "toggle_raw_json", "Raw JSON", key_display="J", show=True), + Binding("j", "jump_primary", "Jump", show=True), + Binding("escape", "dismiss_or_quit", "Back/Quit", show=True), + Binding("q", "quit", "Quit", show=False), + ] + + def __init__(self, params_path: Path): + """Initialize the parameters inspector application. + + Args: + params_path: Filesystem path to the parameters JSON file. + """ + super().__init__() + self.params_path = Path(params_path).resolve() + self.graph = ParamsGraph.from_file(self.params_path) + jname = self.graph.job_path.name if self.graph.job_path else "job" + self.sub_title = f"{jname} ({self.params_path.parent.parent.name if self.graph.job_path else ''})" + + def compose(self) -> ComposeResult: + """Compose top-level layout of the application.""" + yield Header() + with Horizontal(id="mode-bar"): + yield Static(" ⚙️ Config Tree ", id="view-title", classes="title-badge") + yield Static( + " [g] Group · [p] Paths · [J] Raw JSON · [/] Search", + id="mode-hints", + ) + with Horizontal(id="search-bar-container", classes="hidden"): + yield Input(placeholder="Filter parameters...", id="search-input") + yield Static("", id="search-status") + with Horizontal(id="split-container"): + with Vertical(id="tree-container"): + yield ParamsTree(self.graph) + yield NodeDetailPane(self.graph) + yield Footer() + + def on_mount(self) -> None: + """Set initial focus to the parameters tree on mount.""" + self.query_one(ParamsTree).focus() + + def on_tree_node_highlighted(self, event: Tree.NodeHighlighted) -> None: + """Update detail inspector pane when cursor moves over a node. + + Args: + event: Textual NodeHighlighted event. + """ + self.query_one(NodeDetailPane).update_detail(event.node.data) + + def on_tree_node_selected(self, event: Tree.NodeSelected) -> None: + """Activate selected node, jumping to primary definition if shared. + + Args: + event: Textual NodeSelected event. + """ + if event.node.data and event.node.data.target_object_id: + self.action_jump_primary() + + def on_input_changed(self, event: Input.Changed) -> None: + """Filter tree view as query string changes. + + Args: + event: Textual Input.Changed event. + """ + tree = self.query_one(ParamsTree) + tree.set_filter(event.value) + inp = self.query_one("#search-input", Input) + status = self.query_one("#search-status", Static) + hints = self.query_one("#mode-hints", Static) + + q = event.value.strip() + if q: + if tree.match_count == 0: + inp.add_class("not-found") + status.remove_class("matched") + status.add_class("not-found") + status.update(" (not found)") + hints.update( + f" [g] Group · [p] Paths · [/] Search: '{q}' [bold red](not found)[/bold red]" + ) + else: + inp.remove_class("not-found") + status.remove_class("not-found") + status.add_class("matched") + status.update(f" ({tree.match_count} matched)") + hints.update( + f" [g] Group · [p] Paths · [/] Search: '{q}' [green]({tree.match_count} matched)[/green]" + ) + else: + inp.remove_class("not-found") + status.remove_class("not-found") + status.remove_class("matched") + status.update("") + hints.update(" [g] Group · [p] Paths · [J] Raw JSON · [/] Search") + + def on_input_submitted(self, event: Input.Submitted) -> None: + """Hide search bar and return focus to tree upon query submission. + + Args: + event: Textual Input.Submitted event. + """ + self.query_one("#search-bar-container").add_class("hidden") + self.query_one(ParamsTree).focus() + + def action_toggle_grouping(self) -> None: + """Toggle categorized field groups versus flat alphabetical ordering.""" + tree = self.query_one(ParamsTree) + tree.toggle_grouping() + self.notify( + f"Field grouping: {'Categorized' if tree.group_fields else 'Alphabetical'}" + ) + + def action_toggle_paths(self) -> None: + """Toggle absolute filesystem path versus raw relative path display.""" + tree = self.query_one(ParamsTree) + tree.toggle_paths() + self.notify( + f"Path display: {'Absolute' if tree.show_absolute_paths else 'Relative'}" + ) + + def action_toggle_raw_json(self) -> None: + """Toggle raw JSON view versus structured inspector card in detail pane.""" + pane = self.query_one(NodeDetailPane) + pane.toggle_raw_json() + self.notify( + f"Detail pane: {'Raw JSON' if pane.show_raw_json else 'Inspector Card'}" + ) + + def action_expand_all(self) -> None: + """Recursively expand all nodes in the parameter tree.""" + + def _exp(n: TreeNode[NodeData]) -> None: + n.expand() + for c in n.children: + _exp(c) + + _exp(self.query_one(ParamsTree).root) + self.notify("Expanded all") + + def action_collapse_all(self) -> None: + """Collapse all child nodes in the parameter tree.""" + + def _col(n: TreeNode[NodeData]) -> None: + for c in n.children: + _col(c) + c.collapse() + + _col(self.query_one(ParamsTree).root) + self.notify("Collapsed all") + + def action_toggle_search(self) -> None: + """Toggle visibility and focus of the search filter input.""" + box = self.query_one("#search-bar-container") + if "hidden" in box.classes: + box.remove_class("hidden") + self.query_one("#search-input", Input).focus() + else: + box.add_class("hidden") + self.query_one(ParamsTree).focus() + + def action_copy_value(self) -> None: + """Copy the identifier, resolved path, or raw value of the selected node.""" + node = self.query_one(ParamsTree).cursor_node + if not node or not node.data: + return + d = node.data + val = ( + self.graph.objects[d.object_id].identifier + if d.object_id + and d.kind in (NodeKind.OBJECT, NodeKind.ROOT_TASK, NodeKind.SHARED_REF) + else str(d.resolved_path.resolved) + if d.resolved_path + else str(d.field_value) + if d.field_value is not None + else "" + ) + if val: + pyperclip.copy(val) + self.notify(f"Copied: {val[:32]}...") + + def action_copy_python(self) -> None: + """Copy the reproducible Python constructor snippet of the selected object.""" + node = self.query_one(ParamsTree).cursor_node + oid = node.data.object_id if node and node.data else self.graph.root_task_id + if oid and oid in self.graph.objects: + pyperclip.copy(self.graph.to_python_snippet(oid)) + self.notify("Copied Python .C(...) constructor!") + + def action_jump_primary(self) -> None: + """Navigate to the primary definition node in the tree from a shared reference.""" + node = self.query_one(ParamsTree).cursor_node + if not node or not node.data: + return + tid = node.data.target_object_id or node.data.object_id + tree = self.query_one(ParamsTree) + if tid and tid in tree.primary_nodes: + target = tree.primary_nodes[tid] + + # 1. Expand all ancestors from root down to target so it is visible + curr = target.parent + while curr is not None: + if not curr.is_expanded: + curr.expand() + curr = curr.parent + + # 2. Expand target itself so its parameters/fields are revealed + target.expand() + for child in target.children: + if child.data and child.data.kind == NodeKind.CATEGORY: + child.expand() + + # 3. Move cursor to target and scroll into view + tree.move_cursor(target) + tree.scroll_to_node(target) + self.query_one(NodeDetailPane).update_detail(target.data) + self.notify(f"Jumped to primary definition #{tid}") + + def action_dismiss_or_quit(self) -> None: + """Dismiss the search bar if visible, or exit the application.""" + box = self.query_one("#search-bar-container") + if "hidden" not in box.classes: + box.add_class("hidden") + inp = self.query_one("#search-input", Input) + inp.value = "" + inp.remove_class("not-found") + status = self.query_one("#search-status", Static) + status.remove_class("not-found") + status.remove_class("matched") + status.update("") + self.query_one("#mode-hints", Static).update( + " [g] Group · [p] Paths · [J] Raw JSON · [/] Search" + ) + tree = self.query_one(ParamsTree) + tree.set_filter("") + tree.focus() + return + self.exit() diff --git a/src/experimaestro/tools/inspect_params/discovery.py b/src/experimaestro/tools/inspect_params/discovery.py new file mode 100644 index 00000000..066427bf --- /dev/null +++ b/src/experimaestro/tools/inspect_params/discovery.py @@ -0,0 +1,257 @@ +# ruff: noqa: T201 +"""File and workspace discovery utilities for Experimaestro parameters inspector. + +Resolves local workspace paths, remote SSH workspaces (via `experimaestro.utils.remote`), +and locates params.json / experimaestro.json files. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +import sys +from typing import Any, Optional, Union + +from experimaestro.utils.remote import ( + clean_job_target, + find_remote_workspace_file, + resolve_job_file_pattern, +) + +logger = logging.getLogger("experimaestro") + + +def resolve_workspace( + workspace: Optional[str] = None, +) -> Union[Path, Any]: + """Resolve workspace directory from an explicit name/path or Experimaestro default. + + Resolution order: + 1. If workspace argument is provided: + - If it points to an existing local directory, return it. + - Otherwise, look up configured workspace settings via `find_workspace(workspace=...)`. + 2. If no workspace is provided: + - Look up default workspace via `find_workspace()`. + - Check `EXPERIMAESTRO_WORKSPACE` environment variable. + - Search parent directories for `.experimaestro`. + + Args: + workspace: Optional workspace name or filesystem path. + + Returns: + Resolved Path to the local workspace directory, or remote WorkspaceSettings. + + Raises: + FileNotFoundError: If the workspace cannot be determined or does not exist. + """ + if workspace: + p = Path(workspace).expanduser().resolve() + if p.is_dir(): + logger.info("Using workspace %s", p) + return p + try: + from experimaestro.settings import find_workspace + + ws_env = find_workspace(workspace=workspace) + if ws_env: + if ws_env.is_remote: + return ws_env + if ws_env.path: + return Path(ws_env.path).resolve() + except Exception: + pass + raise FileNotFoundError( + f"Specified workspace '{workspace}' could not be found." + ) + + # Try default workspace from experimaestro settings + try: + from experimaestro.settings import find_workspace + + ws_env = find_workspace() + if ws_env: + if ws_env.is_remote: + return ws_env + if ws_env.path: + return Path(ws_env.path).resolve() + except Exception: + pass + + # Fallback: check environment variable or walk up from current working directory + env_ws = os.environ.get("EXPERIMAESTRO_WORKSPACE") + if env_ws and Path(env_ws).is_dir(): + ws_path = Path(env_ws).resolve() + logger.info("Using default workspace %s", ws_path) + return ws_path + + for p in [Path.cwd()] + list(Path.cwd().parents): + if (p / ".experimaestro").is_dir(): + ws_path = p.resolve() + logger.info("Using default workspace %s", ws_path) + return ws_path + + raise FileNotFoundError( + "No workspace specified and default workspace could not be determined." + ) + + +def find_local_workspace_file( + ws_dir: Path, target: str, filename: str = "params.json" +) -> Path: + """Find a specific file within a local workspace job directory. + + Args: + ws_dir: Local workspace root directory. + target: Target job identifier, prefix, or job path. + filename: Relative filename or pattern to locate (default: 'params.json'). + + Returns: + Resolved Path to the matching file. + + Raises: + FileNotFoundError: If target is empty or no matching file exists in workspace. + """ + try: + clean = clean_job_target(target, str(ws_dir), filename=filename) + except ValueError as e: + raise FileNotFoundError( + f"Please specify a job ID or path. {filename} not found in current directory." + ) from e + + pattern = resolve_job_file_pattern(clean, filename) + matches = sorted(list(ws_dir.glob(pattern))) + + if not matches: + raise FileNotFoundError( + f"No job matching '{clean}' found in workspace {ws_dir} (pattern: {pattern})" + ) + + if len(matches) > 1: + print( + f"Warning: Multiple jobs ({len(matches)}) matched '{clean}'. " + f"Using the first match: {matches[0].parent.name} ({matches[0].parent.parent.name})", + file=sys.stderr, + ) + + return matches[0].resolve() + + +def find_workspace_file( + ws_or_path: Any, + target: str, + filename: str = "params.json", + cache_dir: Optional[Path] = None, +) -> Path: + """Find or fetch a file from either a local or remote Experimaestro workspace. + + Args: + ws_or_path: WorkspaceSettings instance (remote/local) or Path to local workspace. + target: Job ID prefix, locator, or job directory path. + filename: Relative filename within the job directory (default: 'params.json'). + cache_dir: Optional local cache directory when fetching from a remote workspace. + + Returns: + Resolved local Path to the found or synchronized file. + """ + from experimaestro.settings import WorkspaceSettings + + if isinstance(ws_or_path, WorkspaceSettings) and ws_or_path.is_remote: + return find_remote_workspace_file( + ws_or_path, target, filename=filename, cache_dir=cache_dir + ) + + ws_dir = ( + ws_or_path.path + if isinstance(ws_or_path, WorkspaceSettings) + else Path(ws_or_path) + ) + return find_local_workspace_file(ws_dir, target, filename=filename) + + +def _find_params_in_path(p: Path) -> Optional[Path]: + """Check if a path points directly to, or contains, a parameters JSON file. + + Checks if `p` is a file directly, or if it is a directory containing + `params.json` or `experimaestro.json` (or any descendant match). + + Args: + p: Path to inspect. + + Returns: + Resolved Path to the found file, or None if no match is found. + """ + if p.is_file(): + return p.resolve() + if p.is_dir(): + for name in ("params.json", "experimaestro.json"): + cand = p / name + if cand.is_file(): + return cand.resolve() + m = list(p.glob("**/params.json")) + list(p.glob("**/experimaestro.json")) + if m: + return m[0].resolve() + return None + + +def find_params_file( + target: Optional[str] = None, workspace: Optional[str] = None +) -> Path: + """Find and resolve the path to a params.json or experimaestro.json file. + + Search resolution order: + 1. If an explicit workspace is specified: + - If remote, query and synchronize via SSH. + - If target is an existing local absolute path, return it directly. + - Otherwise, search inside the specified local workspace. + 2. If no workspace is specified: + - If target is an absolute or user-tilde path, resolve locally; do NOT + attempt workspace discovery. + - If target is an existing local relative path/directory, resolve it. + - Otherwise, discover the default workspace (remote or local) and search + for matching jobs. + + Args: + target: Job ID / hash prefix, path to params.json, or job directory. + workspace: Optional explicit workspace name or path. + + Returns: + Resolved local Path to the parameters JSON file. + + Raises: + FileNotFoundError: If the target cannot be found or matched. + """ + raw = target or "." + p = Path(raw).expanduser() + + # 1. If explicit workspace is specified, check it first (remote overrides local path) + if workspace is not None: + ws_or_path = resolve_workspace(workspace) + from experimaestro.settings import WorkspaceSettings + + if isinstance(ws_or_path, WorkspaceSettings) and ws_or_path.is_remote: + return find_workspace_file(ws_or_path, raw, filename="params.json") + + # Explicit local workspace: if target is an existing local absolute path, return it directly + if p.is_absolute(): + resolved = _find_params_in_path(p) + if resolved: + return resolved + + return find_workspace_file(ws_or_path, raw, filename="params.json") + + # 2. No explicit workspace: check if absolute or tilde path exists locally (never load from workspace) + if p.is_absolute(): + resolved = _find_params_in_path(p) + if resolved: + return resolved + raise FileNotFoundError(f"params.json not found at absolute path: {raw}") + + # 3. Existing relative path locally + resolved = _find_params_in_path(p) + if resolved: + return resolved + + # 4. Otherwise fall back to default workspace discovery + ws_or_path = resolve_workspace() + return find_workspace_file(ws_or_path, raw, filename="params.json") diff --git a/src/experimaestro/tools/inspect_params/models.py b/src/experimaestro/tools/inspect_params/models.py new file mode 100644 index 00000000..c3a9e4e3 --- /dev/null +++ b/src/experimaestro/tools/inspect_params/models.py @@ -0,0 +1,321 @@ +"""Core data models and dependency graph for Experimaestro parameters inspector. + +Represents serialized objects, handles reference traversal, cycle detection, +path resolution, and Python code synthesis. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +import importlib +import json +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +logger = logging.getLogger("experimaestro") + + +@dataclass +class ResolvedPath: + """Representation of a path reference resolved against job or workspace directories. + + Attributes: + raw_value: Original string representation from the parameters JSON. + base: Base reference origin ('job', 'workspace', or 'absolute'). + resolved: Fully resolved filesystem Path. + exists: Whether the file or directory exists on the local filesystem. + is_dir: Whether the resolved path is a directory. + size_bytes: File size in bytes if it exists and is a regular file. + mtime: Last modification timestamp if accessible. + """ + + raw_value: str + base: str # 'job' | 'workspace' | 'absolute' + resolved: Path + exists: bool = False + is_dir: bool = False + size_bytes: Optional[int] = None + mtime: Optional[datetime] = None + + @classmethod + def resolve( + cls, raw_val: str, base: str, job_dir: Optional[Path], ws_dir: Optional[Path] + ) -> ResolvedPath: + """Resolve a raw path string relative to job or workspace directory. + + Resolution precedence: + 1. If base is 'job' and job_dir is provided: resolve relative to job_dir. + 2. If base is 'workspace' and ws_dir is provided: resolve relative to ws_dir. + 3. If raw path is relative and job_dir is provided: resolve relative to job_dir. + 4. Otherwise, resolve raw path directly. + """ + p = Path(raw_val) + res = ( + (job_dir / p).resolve() + if base == "job" and job_dir + else (ws_dir / p).resolve() + if base == "workspace" and ws_dir + else (job_dir / p).resolve() + if not p.is_absolute() and job_dir + else p.resolve() + ) + exists = res.exists() + is_dir = res.is_dir() if exists else False + size, mtime = None, None + if exists and not is_dir: + try: + st = res.stat() + size, mtime = st.st_size, datetime.fromtimestamp(st.st_mtime) + except OSError: + pass + return cls(raw_val, base, res, exists, is_dir, size, mtime) + + +@dataclass +class ObjectNode: + """A deserialized configuration object or task within the Experimaestro graph. + + Attributes: + id: Unique numeric object identifier. + module: Python module declaring the object's class. + type_name: Short class name of the object. + qualified_name: Full module.class or qualified typename. + identifier: Content hash or task identifier string. + is_task: True if this object represents an executable task. + is_root_task: True if this is the primary root task of the job. + init_tasks: IDs of prerequisite tasks that must run before this task. + fields: Key-value mapping of parameters, sub-configs, or paths. + incoming_refs: List of (source_object_id, field_name) referencing this object. + outgoing_refs: List of (field_name, target_object_id) referenced by this object. + docstring: Optional docstring extracted from the underlying Python class. + """ + + id: int + module: str + type_name: str + qualified_name: str + identifier: str + is_task: bool + is_root_task: bool + init_tasks: List[int] + fields: Dict[str, Any] + incoming_refs: List[Tuple[int, str]] = field(default_factory=list) + outgoing_refs: List[Tuple[str, int]] = field(default_factory=list) + docstring: Optional[str] = None + + +class ParamsGraph: + """Parses, indexes, and queries the object dependency graph of an Experimaestro job. + + Supports both single-job dictionaries (`params.json`) and multi-task list + payloads (such as Hugging Face Hub `experimaestro.json` exports). + + Attributes: + raw_data: Original deserialized JSON data. + job_path: Path to the directory containing the parameters file. + workspace_path: Resolved path to the enclosing workspace directory. + objects: Mapping of numeric object IDs to ObjectNode instances. + root_task_id: Object ID of the primary root task, if determined. + version: Schema version number. + experimaestro_version: Experimaestro version string. + carbon: Carbon emission tracking configuration dictionary. + """ + + workspace_path: Optional[Path] + + def __init__( + self, + data: Union[Dict[str, Any], List[Dict[str, Any]]], + job_path: Optional[Path] = None, + workspace_path: Optional[Path] = None, + ): + """Initialize the graph from a dictionary or list of raw object specifications. + + Args: + data: Deserialized params.json dict or experimaestro.json list. + job_path: Optional filesystem path to the enclosing job directory. + workspace_path: Optional workspace path (auto-discovered if omitted). + """ + self.raw_data = data + self.job_path = job_path.resolve() if job_path else None + if isinstance(data, list): + ws = None + self.version = 0 + self.experimaestro_version = "unknown" + self.carbon = {} + raw_objs = data + else: + ws = data.get("workspace") + self.version = data.get("version", 0) + self.experimaestro_version = data.get("experimaestro", "unknown") + self.carbon = data.get("carbon", {}) + raw_objs = data.get("objects", []) + + self.workspace_path = ( + Path(ws).resolve() + if ws + else workspace_path.resolve() + if workspace_path + else self._find_ws(self.job_path) + ) + self.objects: Dict[int, ObjectNode] = {} + self.root_task_id: Optional[int] = None + self._parse(raw_objs) + + @staticmethod + def _find_ws(path: Optional[Path]) -> Optional[Path]: + """Search upwards from a directory for the nearest `.experimaestro` root.""" + if not path: + return None + for p in [path] + list(path.parents): + if (p / ".experimaestro").is_dir(): + return p + return None + + @classmethod + def from_file(cls, path: Path) -> ParamsGraph: + """Load a parameter graph from a file or directory. + + Accepts direct paths to `params.json` or `experimaestro.json`, or a directory + containing either file. + """ + p = path.resolve() + if p.is_dir(): + if (p / "params.json").is_file(): + f = p / "params.json" + elif (p / "experimaestro.json").is_file(): + f = p / "experimaestro.json" + else: + f = p / "params.json" + else: + f = p + if not f.exists(): + raise FileNotFoundError(f"params.json not found at: {f}") + return cls(json.loads(f.read_text(encoding="utf-8")), job_path=f.parent) + + def _parse(self, raw_objs: List[Dict[str, Any]]) -> None: + """Construct ObjectNode instances and resolve reference graphs.""" + target_hash = self.job_path.name if self.job_path else None + cand_root = None + + for o in raw_objs: + oid = int(o["id"]) + mod, typ = o.get("module", ""), o.get("type", "") + qname = o.get("typename") or f"{mod}.{typ}" if mod else typ or str(oid) + ident = o.get("identifier", "") + inits = [int(x) for x in o.get("init-tasks", [])] + t_ptr = o.get("task") + is_task = (t_ptr is not None and int(t_ptr) == oid) or bool(inits) + + doc = None + if mod and typ: + try: + cls_obj = getattr(importlib.import_module(mod), typ, None) + doc = ( + cls_obj.__doc__.strip() if cls_obj and cls_obj.__doc__ else None + ) + except Exception: + pass + + node = ObjectNode( + oid, + mod, + typ, + qname, + ident, + is_task, + False, + inits, + o.get("fields", {}), + docstring=doc, + ) + self.objects[oid] = node + if target_hash and ident == target_hash: + cand_root = oid + + self.root_task_id = ( + cand_root if cand_root else (int(raw_objs[-1]["id"]) if raw_objs else None) + ) + if self.root_task_id and self.root_task_id in self.objects: + self.objects[self.root_task_id].is_root_task = True + self.objects[self.root_task_id].is_task = True + + def _extract_refs(val: Any, prefix: str = "") -> List[Tuple[str, int]]: + refs: List[Tuple[str, int]] = [] + if isinstance(val, dict): + if val.get("type") == "python" and "value" in val: + refs.append((prefix, int(val["value"]))) + else: + for k_sub, v_sub in val.items(): + sub_p = f"{prefix}.{k_sub}" if prefix else k_sub + refs.extend(_extract_refs(v_sub, sub_p)) + elif isinstance(val, list): + for idx, item in enumerate(val): + sub_p = f"{prefix}[{idx}]" + refs.extend(_extract_refs(item, sub_p)) + return refs + + for oid, n in self.objects.items(): + for it in n.init_tasks: + if it in self.objects: + n.outgoing_refs.append(("__init_task__", it)) + self.objects[it].incoming_refs.append((oid, "__init_task__")) + for k, v in n.fields.items(): + for rk, tid in _extract_refs(v, k): + if tid in self.objects: + n.outgoing_refs.append((rk, tid)) + self.objects[tid].incoming_refs.append((oid, rk)) + + def resolve_path(self, spec: Any) -> Optional[ResolvedPath]: + """Attempt to resolve a parameter value as a path specification.""" + if isinstance(spec, dict) and spec.get("type") in ("path", "path.serialized"): + return ResolvedPath.resolve( + spec.get("value", ""), + spec.get("base", "absolute"), + self.job_path, + self.workspace_path, + ) + return None + + def to_python_snippet( + self, oid: int, indent: int = 0, seen: Optional[Set[int]] = None + ) -> str: + """Synthesize a reproducible Python `.C(...)` configuration snippet.""" + seen = seen or set() + node = self.objects.get(oid) + if not node: + return f"#{oid}" + if oid in seen: + return f"{node.type_name}.C(...) # cycle to #{oid}" + seen.add(oid) + + sp, spi = " " * indent, " " * (indent + 1) + args = [ + f"{spi}{k}={self._fmt_py(v, indent + 1, set(seen))}," + for k, v in sorted(node.fields.items()) + ] + return f"{node.type_name}.C(\n" + "\n".join(args) + f"\n{sp})" + + def _fmt_py(self, v: Any, indent: int, seen: Set[int]) -> str: + """Format an individual field value for Python code synthesis.""" + if isinstance(v, dict): + t = v.get("type") + if t == "python": + return self.to_python_snippet(int(v["value"]), indent, seen) + if t in ("path", "path.serialized"): + b = v.get("base") + return f'Path("{v.get("value")}")' + (f" # base: {b}" if b else "") + if t == "enum": + return f"{v.get('enum')}.{v.get('value')}" + return ( + "{" + + ", ".join( + f'"{k}": {self._fmt_py(val, indent, seen)}' for k, val in v.items() + ) + + "}" + ) + if isinstance(v, list): + return "[" + ", ".join(self._fmt_py(x, indent, seen) for x in v) + "]" + return repr(v) diff --git a/src/experimaestro/tools/xpm_dag/xpm_dag.py b/src/experimaestro/tools/xpm_dag/xpm_dag.py index cff71845..0ea5a82b 100644 --- a/src/experimaestro/tools/xpm_dag/xpm_dag.py +++ b/src/experimaestro/tools/xpm_dag/xpm_dag.py @@ -50,6 +50,15 @@ def resolve_run_dir( else: ws_env = find_workspace(experiment_id=experiment_id) + if getattr(ws_env, "is_remote", False): + from experimaestro.utils.remote import find_remote_workspace_experiment_run + + try: + return find_remote_workspace_experiment_run(ws_env, experiment_id, run_id) + except Exception as e: + logging.error("Failed to resolve remote experiment run: %s", e) + return None + workspace = ws_env.path experiments_dir = workspace / "experiments" if not experiments_dir.exists(): diff --git a/src/experimaestro/utils/remote.py b/src/experimaestro/utils/remote.py new file mode 100644 index 00000000..00934dab --- /dev/null +++ b/src/experimaestro/utils/remote.py @@ -0,0 +1,383 @@ +"""Remote workspace utilities via SSH and Rsync. + +Provides lightweight remote querying, pattern matching, and file synchronization +to support tools like `experimaestro inspect` and `experimaestro dag` on remote SSH workspaces. +""" + +from __future__ import annotations + +import atexit +import logging +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +import tempfile +from typing import Any, List, Optional, Tuple + +logger = logging.getLogger("experimaestro") + +_SESSION_REMOTE_CACHE_DIR: Optional[Path] = None + + +def get_session_cache_dir() -> Path: + """Return a managed temporary directory for cached remote workspace files. + + Creates the directory lazily on first access and registers automatic cleanup + upon process termination via atexit. + + Returns: + Path to the session temporary cache directory. + """ + global _SESSION_REMOTE_CACHE_DIR + if _SESSION_REMOTE_CACHE_DIR is None or not _SESSION_REMOTE_CACHE_DIR.exists(): + tmp = tempfile.mkdtemp(prefix="xpm_cache_") + _SESSION_REMOTE_CACHE_DIR = Path(tmp) + atexit.register(shutil.rmtree, tmp, ignore_errors=True) + return _SESSION_REMOTE_CACHE_DIR + + +def clean_job_target( + raw: str, ws_base: str = "", filename: Optional[str] = None +) -> str: + """Normalize a target job locator, prefix, or path. + + Strips surrounding slashes, explicit file names, workspace root prefixes, + and leading 'jobs/' prefixes. + + Args: + raw: Raw input target string. + ws_base: Optional workspace base path to strip. + filename: Optional filename to strip from the end (e.g. 'params.json'). + + Returns: + Normalized target string (e.g. 'ee76fb8f' or 'my_task/ee76fb8f'). + + Raises: + ValueError: If the normalized target is empty or root. + """ + clean = raw.strip("/") + if filename: + clean = clean.removesuffix(f"/{filename}").removesuffix(filename).strip("/") + else: + for ext in (".json", ".log", ".txt", ".out", ".err"): + if clean.endswith(ext): + clean = str(Path(clean).parent) + break + + ws_clean = ws_base.strip("/") + if ws_clean and clean.startswith(ws_clean): + clean = clean[len(ws_clean) :].strip("/") + + if "/jobs/" in clean: + clean = clean.split("/jobs/", 1)[1].strip("/") + elif clean.startswith("jobs/"): + clean = clean[len("jobs/") :].strip("/") + + if not clean or clean == ".": + raise ValueError("Job target cannot be empty or root.") + + return clean + + +def resolve_job_file_pattern(clean_target: str, filename: str) -> str: + """Construct workspace-relative glob pattern for a target and filename. + + Args: + clean_target: Normalized target (e.g. 'my_task/ee76fb8f' or 'ee76fb8f'). + filename: Relative filename to find (e.g. 'params.json', 'experimaestro.json'). + + Returns: + Workspace-relative glob pattern string under the jobs directory. + """ + clean_target = clean_target.replace(":", "/") + if "/" in clean_target: + return f"jobs/{clean_target}*/{filename}" + return f"jobs/*/{clean_target}*/{filename}" + + +def build_ssh_cmd(ssh_settings: Any, remote_command: str) -> List[str]: + """Construct an SSH command list respecting options and shell initialization. + + Args: + ssh_settings: SSHSettings object with host, options, and shell_init. + remote_command: Command string to execute on the remote host. + + Returns: + Command list suitable for subprocess.run. + """ + cmd = ["ssh"] + if ssh_settings.options: + cmd.extend(ssh_settings.options) + cmd.append(ssh_settings.host) + + full_remote_cmd = remote_command + if ssh_settings.shell_init: + full_remote_cmd = f"{ssh_settings.shell_init}; {remote_command}" + + cmd.append(full_remote_cmd) + return cmd + + +def run_ssh_command( + ssh_settings: Any, remote_command: str +) -> subprocess.CompletedProcess: + """Execute a command string on the remote host via SSH with captured output. + + Args: + ssh_settings: SSHSettings instance. + remote_command: Command string to run remotely. + + Returns: + CompletedProcess instance with stdout and stderr. + """ + cmd = build_ssh_cmd(ssh_settings, remote_command) + return subprocess.run(cmd, capture_output=True, text=True) + + +def fetch_remote_file_rsync( + ssh_settings: Any, remote_path: str, local_dest: Path +) -> None: + """Transfer a single remote file to local_dest using rsync or scp. + + Args: + ssh_settings: SSHSettings object with host and options. + remote_path: Absolute or workspace path on the remote host. + local_dest: Local Path where the file should be written. + + Raises: + RuntimeError: If both rsync and scp transfer attempts fail. + """ + local_dest.parent.mkdir(parents=True, exist_ok=True) + source = f"{ssh_settings.host}:{remote_path}" + dest = str(local_dest) + + cmd = ["rsync", "-L", "-a"] + if ssh_settings.options: + ssh_opt_str = " ".join(shlex.quote(opt) for opt in ssh_settings.options) + cmd.extend(["-e", f"ssh {ssh_opt_str}"]) + cmd.extend([source, dest]) + + res = subprocess.run(cmd, capture_output=True, text=True) + if res.returncode != 0: + # Fallback to scp if rsync failed or is unavailable + scp_cmd = ["scp"] + if ssh_settings.options: + scp_cmd.extend(ssh_settings.options) + scp_cmd.extend([source, dest]) + scp_res = subprocess.run(scp_cmd, capture_output=True, text=True) + if scp_res.returncode != 0: + err = res.stderr.strip() or scp_res.stderr.strip() + raise RuntimeError( + f"Failed to transfer {remote_path} from {ssh_settings.host}: {err}" + ) + + +def fetch_remote_files_rsync( + ssh_settings: Any, + remote_src_dir: str, + local_dest_dir: Path, + includes: List[str], +) -> None: + """Transfer specific files matching includes from remote directory using rsync. + + Args: + ssh_settings: SSHSettings object. + remote_src_dir: Remote source directory path. + local_dest_dir: Local destination directory. + includes: List of filename patterns to include (e.g. ['objects.jsonl', 'jobs.jsonl']). + """ + local_dest_dir.mkdir(parents=True, exist_ok=True) + source = f"{ssh_settings.host}:{remote_src_dir.rstrip('/')}/" + dest = f"{str(local_dest_dir).rstrip('/')}/" + + cmd = ["rsync", "-L", "-a"] + if ssh_settings.options: + ssh_opt_str = " ".join(shlex.quote(opt) for opt in ssh_settings.options) + cmd.extend(["-e", f"ssh {ssh_opt_str}"]) + + for inc in includes: + cmd.extend(["--include", inc]) + cmd.extend(["--exclude", "*", source, dest]) + + res = subprocess.run(cmd, capture_output=True, text=True) + if res.returncode != 0: + # Fallback to pulling files individually + for inc in includes: + remote_file = f"{remote_src_dir.rstrip('/')}/{inc}" + local_file = local_dest_dir / inc + try: + fetch_remote_file_rsync(ssh_settings, remote_file, local_file) + except Exception: + pass + + +def find_remote_workspace_file( + ws: Any, + target: str, + filename: str = "params.json", + cache_dir: Optional[Path] = None, +) -> Path: + """Fetch a file from a remote workspace job directory using SSH and Rsync. + + Resolves remote workspace job directories matching `target` and `filename` + via SSH pattern matching, and synchronizes the file into a local cache directory. + + Args: + ws: Remote WorkspaceSettings instance with SSH configuration. + target: Job ID / hash prefix or remote job directory subpath. + filename: Relative filename to fetch (default: 'params.json'). + cache_dir: Optional local directory to store fetched files. If None, + uses the managed session temporary cache directory. + + Returns: + Local Path to the synchronized file. + + Raises: + FileNotFoundError: If target is empty, or no matching file exists remotely. + RuntimeError: If remote SSH command execution or file transfer fails. + """ + ws_raw = str(getattr(ws, "_raw_path", "")).strip("/") + try: + clean_target = clean_job_target(target, ws_raw, filename=filename) + except ValueError as e: + raise FileNotFoundError( + f"Please specify a job ID or path for remote workspace '{ws.id}'." + ) from e + + pattern = resolve_job_file_pattern(clean_target, filename) + remote_pattern = f"{ws._raw_path}/{pattern}" + + logger.info( + "Querying remote workspace %s via SSH for %s matching '%s'...", + ws.id, + filename, + clean_target, + ) + + py_bin = ws.ssh.python or "python3" + py_script = "import glob, sys; [print(p) for p in glob.glob(sys.argv[1])]" + remote_cmd = f"{py_bin} -c {py_script!r} {shlex.quote(remote_pattern)}" + + res = run_ssh_command(ws.ssh, remote_cmd) + if res.returncode != 0: + err = res.stderr.strip() or res.stdout.strip() + raise RuntimeError(f"Failed to query remote workspace '{ws.id}' via SSH: {err}") + + matches = [line.strip() for line in res.stdout.splitlines() if line.strip()] + if not matches: + raise FileNotFoundError( + f"No job matching '{clean_target}' found in remote workspace '{ws.id}' ({ws._raw_path})" + ) + + if len(matches) > 1: + print( # noqa: T201 + f"Warning: Multiple jobs ({len(matches)}) matched '{clean_target}'. " + f"Using the first match: {matches[0]}", + file=sys.stderr, + ) + + remote_file_path = matches[0] + rel_path = remote_file_path[len(ws._raw_path) :].lstrip("/") + dest_cache = cache_dir or get_session_cache_dir() + local_dest = dest_cache / rel_path + + logger.info("Synchronizing %s from %s into %s...", rel_path, ws.id, local_dest) + fetch_remote_file_rsync(ws.ssh, remote_file_path, local_dest) + + if not local_dest.exists(): + raise FileNotFoundError( + f"{filename} not found in local cache {local_dest} after transfer." + ) + + return local_dest + + +fetch_remote_workspace_file = find_remote_workspace_file + + +def find_remote_workspace_experiment_run( + ws: Any, + experiment_id: str, + run_id: Optional[str] = None, + cache_dir: Optional[Path] = None, +) -> Tuple[str, Path]: + """Find and synchronize an experiment run's metadata from a remote workspace. + + Queries the remote workspace for available runs in `experiments//`, + selects the requested run (or the most recent one), and pulls `objects.jsonl`, + `jobs.jsonl`, and `progress.jsonl` into the local session cache directory. + + Args: + ws: Remote WorkspaceSettings instance. + experiment_id: Experiment identifier. + run_id: Optional specific run timestamp ID. If None, picks the latest. + cache_dir: Optional cache directory. + + Returns: + Tuple of (selected_run_id, local_run_dir_path). + + Raises: + FileNotFoundError: If experiment or run cannot be found remotely. + RuntimeError: If remote SSH command execution fails. + """ + ws_raw = str(getattr(ws, "_raw_path", "")).rstrip("/") + exp_remote_path = f"{ws_raw}/experiments/{experiment_id}" + + py_bin = ws.ssh.python or "python3" + # List run directories under the experiment directory + py_script = ( + "import os, sys, glob; " + "d = sys.argv[1]; " + "runs = [os.path.basename(p) for p in glob.glob(os.path.join(d, '*')) if os.path.isdir(p)]; " + "print('\\n'.join(sorted(runs)))" + ) + remote_cmd = f"{py_bin} -c {py_script!r} {shlex.quote(exp_remote_path)}" + + res = run_ssh_command(ws.ssh, remote_cmd) + if res.returncode != 0: + err = res.stderr.strip() or res.stdout.strip() + raise RuntimeError( + f"Failed to query experiment '{experiment_id}' in remote workspace '{ws.id}': {err}" + ) + + available_runs = [line.strip() for line in res.stdout.splitlines() if line.strip()] + if not available_runs: + raise FileNotFoundError( + f"No runs found for experiment '{experiment_id}' in remote workspace '{ws.id}'." + ) + + if run_id: + if run_id not in available_runs: + raise FileNotFoundError( + f"Run '{run_id}' not found for experiment '{experiment_id}' in remote workspace '{ws.id}'. " + f"Available runs: {available_runs}" + ) + selected_run_id = run_id + else: + selected_run_id = available_runs[-1] + + remote_run_dir = f"{exp_remote_path}/{selected_run_id}" + dest_cache = cache_dir or get_session_cache_dir() + local_run_dir = dest_cache / "experiments" / experiment_id / selected_run_id + local_run_dir.mkdir(parents=True, exist_ok=True) + + logger.info( + "Synchronizing DAG files for run %s (%s) from %s into %s...", + selected_run_id, + experiment_id, + ws.id, + local_run_dir, + ) + + # Fetch lightweight metadata files needed for DAG visualization + includes = ["objects.jsonl", "jobs.jsonl", "progress.jsonl", "run.yaml"] + fetch_remote_files_rsync(ws.ssh, remote_run_dir, local_run_dir, includes) + + if not (local_run_dir / "objects.jsonl").exists(): + # Fallback: pull objects.jsonl individually to verify + remote_obj = f"{remote_run_dir}/objects.jsonl" + fetch_remote_file_rsync(ws.ssh, remote_obj, local_run_dir / "objects.jsonl") + + return selected_run_id, local_run_dir From 39a151fc5eac31a7d9c677f1b108fe0a2e5d2dfd Mon Sep 17 00:00:00 2001 From: Victor Morand Date: Mon, 19 Oct 2026 16:18:38 +0200 Subject: [PATCH 4/7] feat: add param inspector in the monitoring TUI --- src/experimaestro/scheduler/state_provider.py | 39 +- src/experimaestro/tests/test_tui_inspect.py | 273 +++++++++++++ .../tools/inspect_params/__init__.py | 2 + src/experimaestro/tools/inspect_params/app.py | 363 +++++++++++++----- src/experimaestro/tui/__init__.py | 5 +- src/experimaestro/tui/app.py | 47 +++ src/experimaestro/tui/app.tcss | 92 +++++ src/experimaestro/tui/dialogs.py | 1 + src/experimaestro/tui/messages.py | 20 +- src/experimaestro/tui/widgets/jobs.py | 61 ++- 10 files changed, 793 insertions(+), 110 deletions(-) create mode 100644 src/experimaestro/tests/test_tui_inspect.py diff --git a/src/experimaestro/scheduler/state_provider.py b/src/experimaestro/scheduler/state_provider.py index 9609882e..9ce29f27 100644 --- a/src/experimaestro/scheduler/state_provider.py +++ b/src/experimaestro/scheduler/state_provider.py @@ -535,13 +535,50 @@ def get_scheduler_log_path(self) -> Optional[Path]: """ return None - def sync_path(self, path: str) -> Optional[Path]: + def sync_path(self, path: str, include: list[str] | None = None) -> Optional[Path]: """Sync a specific path from remote (remote providers only) Returns None for local providers or if sync fails. """ return None + def has_job_params(self, job: BaseJob) -> bool: + """Check if parameter definitions exist for a job.""" + if not job or not job.path: + return False + if self.is_remote: + return True + return (job.path / "params.json").is_file() or ( + job.path / "experimaestro.json" + ).is_file() + + def get_job_params_file(self, job: BaseJob) -> Optional[Path]: + """Get the local path to a job's parameters file (params.json or experimaestro.json). + + For local providers, checks the job directory on disk. + For remote providers, synchronizes params.json or experimaestro.json to the local + cache directory using sync_path and returns the local Path. + """ + if not job or not job.path: + return None + + if self.is_remote: + local_dir = self.sync_path( + str(job.path), include=["params.json", "experimaestro.json"] + ) + if local_dir: + for filename in ("params.json", "experimaestro.json"): + cand = local_dir / filename + if cand.is_file(): + return cand + return None + + for filename in ("params.json", "experimaestro.json"): + cand = job.path / filename + if cand.is_file(): + return cand + return None + def get_orphan_jobs(self, *, refresh: bool = False) -> List[BaseJob]: """Get orphan jobs (jobs not associated with any experiment run)""" return [] diff --git a/src/experimaestro/tests/test_tui_inspect.py b/src/experimaestro/tests/test_tui_inspect.py new file mode 100644 index 00000000..258f0404 --- /dev/null +++ b/src/experimaestro/tests/test_tui_inspect.py @@ -0,0 +1,273 @@ +"""Tests for parameter inspection integration in the TUI monitor app.""" + +from pathlib import Path +from unittest.mock import MagicMock +import pytest +from textual.app import App, ComposeResult +from textual.widgets import Label, Static + +from experimaestro.scheduler.interfaces import BaseJob, JobState +from experimaestro.scheduler.state_provider import StateProvider +from experimaestro.tools.inspect_params import ParamsInspectorScreen, ParamsTree +from experimaestro.tui.messages import InspectJobParams, InspectJobParamsRequest +from experimaestro.tui.widgets.jobs import ( + JobDetailView, + JobsTable, + JobsTreeView, + TreeJobData, +) + + +class DummyJob(BaseJob): + """Minimal dummy job for state provider and widget testing""" + + def __init__(self, identifier: str, task_id: str, path: Path): + self.identifier = identifier + self.task_id = task_id + self.path = path + self._state = JobState.RUNNING + self.starttime = None + self.endtime = None + self.progress = [] + self._process = None + self.carbon_metrics = None + + +class DummyLocalStateProvider(StateProvider): + """Minimal local state provider for testing""" + + @property + def is_remote(self) -> bool: + return False + + def get_display_path(self, job): + return str(job.path) + + def get_process_info(self, job): + return None + + def get_experiments(self, *args, **kwargs): + return [] + + def get_experiment(self, *args, **kwargs): + return None + + def get_experiment_runs(self, *args, **kwargs): + return [] + + def get_jobs(self, *args, **kwargs): + return [] + + def get_job(self, task_id: str, job_id: str, *args, **kwargs): + return None + + def get_all_jobs(self, *args, **kwargs): + return [] + + def get_services(self, *args, **kwargs): + return [] + + def get_tags_map(self, *args, **kwargs): + return {} + + def get_dependencies_map(self, *args, **kwargs): + return {} + + def get_current_run(self, *args, **kwargs): + return None + + def get_experiment_job_info(self, *args, **kwargs): + return {} + + def kill_job(self, *args, **kwargs): + return True + + def clean_job(self, *args, **kwargs): + return True + + def close(self): + pass + + +class DummyRemoteStateProvider(DummyLocalStateProvider): + """Minimal remote state provider for testing""" + + @property + def is_remote(self) -> bool: + return True + + +def test_state_provider_job_params_local(tmp_path: Path): + """Verify local StateProvider parameter detection and retrieval.""" + sp = DummyLocalStateProvider() + job_dir = tmp_path / "job_a" + job_dir.mkdir() + job = DummyJob("job_a", "task_pkg.task_a", job_dir) + + # Neither exists + assert sp.has_job_params(job) is False + assert sp.get_job_params_file(job) is None + + # params.json exists + params_file = job_dir / "params.json" + params_file.write_text('{"version": 1}') + assert sp.has_job_params(job) is True + assert sp.get_job_params_file(job) == params_file + + # experimaestro.json fallback + params_file.unlink() + xpm_file = job_dir / "experimaestro.json" + xpm_file.write_text('[{"version": 1}]') + assert sp.has_job_params(job) is True + assert sp.get_job_params_file(job) == xpm_file + + +def test_state_provider_job_params_remote(tmp_path: Path): + """Verify remote StateProvider synchronizes params via sync_path.""" + sp = DummyRemoteStateProvider() + remote_job_dir = Path("/remote/workspace/jobs/task_a/12345678") + job = DummyJob("12345678", "task_a", remote_job_dir) + + # Remote job always has_job_params = True if path is set + assert sp.has_job_params(job) is True + + # Mock sync_path + cache_dir = tmp_path / "cache_job" + cache_dir.mkdir() + cached_params = cache_dir / "params.json" + cached_params.write_text('{"version": 1}') + + sp.sync_path = MagicMock(return_value=cache_dir) + + result_path = sp.get_job_params_file(job) + sp.sync_path.assert_called_once_with( + str(remote_job_dir), include=["params.json", "experimaestro.json"] + ) + assert result_path == cached_params + + +def test_job_detail_view_hints_and_actions(tmp_path: Path): + """Verify JobDetailView updates the hint and posts InspectJobParams.""" + sp = DummyLocalStateProvider() + view = JobDetailView.__new__(JobDetailView) + view.state_provider = sp + view.tags_map = {} + view.dependencies_map = {} + view.job_task_id_map = {} + + labels: dict[str, str] = {} + + def mock_query_one(selector, expected_type=None): + mock_label = MagicMock(spec=Label) + mock_label.update = lambda text: labels.__setitem__(selector, str(text)) + return mock_label + + view.query_one = mock_query_one + + # Setup dummy job with both logs and params + job_dir = tmp_path / "job_both" + job_dir.mkdir() + (job_dir / "params.json").write_text("{}") + (job_dir / "my_task.out").write_text("log") + job = DummyJob("job_both", "pkg.my_task", job_dir) + view.job_data = job + + view._update_job_display(job) + assert "Press 'l' to view logs, 'i' to inspect params" in labels["#job-logs-hint"] + + # Action posts InspectJobParams + messages = [] + view.post_message = lambda msg: messages.append(msg) + view.action_inspect_params() + assert len(messages) == 1 + assert isinstance(messages[0], InspectJobParams) + assert messages[0].job is job + + # Setup dummy job with only params + (job_dir / "my_task.out").unlink() + view._update_job_display(job) + assert "Press 'i' to inspect params" in labels["#job-logs-hint"] + assert "no logs yet" in labels["#job-logs-hint"] + + # Setup dummy job with only logs + (job_dir / "params.json").unlink() + (job_dir / "my_task.out").write_text("log") + view._update_job_display(job) + assert labels["#job-logs-hint"] == "[bold cyan]Press 'l' to view logs[/bold cyan]" + + # Setup dummy job with neither + (job_dir / "my_task.out").unlink() + view._update_job_display(job) + assert labels["#job-logs-hint"] == "(no log or param files found)" + + +def test_jobs_table_inspect_action(): + """Verify JobsTable action_inspect_params posts InspectJobParamsRequest.""" + table = JobsTable.__new__(JobsTable) + table.current_experiment = "mnist_train" + table._tree_mode = False + table._get_selected_job_info = lambda: ("job_123", "pkg.my_task") + + messages = [] + table.post_message = lambda msg: messages.append(msg) + table.action_inspect_params() + + assert len(messages) == 1 + assert isinstance(messages[0], InspectJobParamsRequest) + assert messages[0].job_id == "job_123" + assert messages[0].task_id == "pkg.my_task" + assert messages[0].experiment_id == "mnist_train" + + +def test_jobs_tree_inspect_action(): + """Verify JobsTreeView action_inspect_params posts InspectJobParamsRequest.""" + tree_view = JobsTreeView.__new__(JobsTreeView) + tree_view.current_experiment = "mnist_train" + tree_view._get_selected_job_data = lambda: TreeJobData( + job_id="job_456", task_id="pkg.eval_task", status_text="RUNNING", tags={} + ) + + messages = [] + tree_view.post_message = lambda msg: messages.append(msg) + tree_view.action_inspect_params() + + assert len(messages) == 1 + assert isinstance(messages[0], InspectJobParamsRequest) + assert messages[0].job_id == "job_456" + assert messages[0].task_id == "pkg.eval_task" + assert messages[0].experiment_id == "mnist_train" + + +@pytest.mark.anyio +async def test_params_inspector_screen_pushed_and_dismissed(tmp_path: Path): + """Verify ParamsInspectorScreen can be pushed onto an App and dismissed with escape.""" + params_file = tmp_path / "params.json" + params_file.write_text( + '{"version": 1, "task": "test_task", "params": {"lr": 0.01}}' + ) + + class ParentApp(App): + def compose(self) -> ComposeResult: + yield Static("Parent Screen", id="parent-text") + + app = ParentApp() + async with app.run_test() as pilot: + assert len(app.screen_stack) == 1 + assert app.query_one("#parent-text", Static) is not None + + # Push the screen as ExperimaestroUI would + screen = ParamsInspectorScreen(params_file, title="Test Job") + app.push_screen(screen) + await pilot.pause() + + assert len(app.screen_stack) == 2 + # Tree is active on the pushed screen + tree = app.screen.query_one(ParamsTree) + assert tree is not None + + # Press escape to dismiss back to parent screen + await pilot.press("escape") + await pilot.pause() + + assert len(app.screen_stack) == 1 + assert app.query_one("#parent-text", Static) is not None diff --git a/src/experimaestro/tools/inspect_params/__init__.py b/src/experimaestro/tools/inspect_params/__init__.py index 8f11443b..77168cdc 100644 --- a/src/experimaestro/tools/inspect_params/__init__.py +++ b/src/experimaestro/tools/inspect_params/__init__.py @@ -13,6 +13,7 @@ NodeDetailPane, NodeKind, ParamsInspectorApp, + ParamsInspectorScreen, ParamsTree, ) from experimaestro.tools.inspect_params.discovery import ( @@ -43,6 +44,7 @@ "ObjectNode", "ParamsGraph", "ParamsInspectorApp", + "ParamsInspectorScreen", "ParamsTree", "ResolvedPath", "clean_job_target", diff --git a/src/experimaestro/tools/inspect_params/app.py b/src/experimaestro/tools/inspect_params/app.py index b62f0cc3..457fb6e7 100644 --- a/src/experimaestro/tools/inspect_params/app.py +++ b/src/experimaestro/tools/inspect_params/app.py @@ -8,7 +8,7 @@ import json import logging from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple import pyperclip # type: ignore[import-untyped] from rich.console import Group @@ -20,13 +20,20 @@ from textual.app import App, ComposeResult from textual.binding import Binding from textual.containers import Horizontal, Vertical, VerticalScroll +from textual.screen import Screen +from textual import work from textual.widgets import Footer, Header, Input, Static, Tree from textual.widgets.tree import TreeNode +if TYPE_CHECKING: + from experimaestro.scheduler.interfaces import BaseJob + from experimaestro.scheduler.state_provider import StateProvider + from experimaestro.tools.inspect_params.models import ( ParamsGraph, ResolvedPath, ) +from experimaestro.tui import APP_TCSS_PATH logger = logging.getLogger("experimaestro") @@ -133,7 +140,7 @@ def _apply_highlight(self, text: Text) -> None: if self.filter_query: text.highlight_words( [self.filter_query], - style="bold black on #f9e2af", + style="bold black on yellow", case_sensitive=False, ) @@ -790,33 +797,21 @@ def _make_table(rows: List[Tuple[str, str]], title: Optional[str] = None) -> Any # --------------------------------------------------------------------------- -class ParamsInspectorApp(App): - """Standalone Textual App for inspecting experimaestro params.json. +# --------------------------------------------------------------------------- +# Inspector Screen & App +# --------------------------------------------------------------------------- - Features interactive tree navigation, categorization, path resolution, - fuzzy search filtering, Python snippet copying, and raw JSON inspection. - """ - TITLE = "Experimaestro Parameters Inspector" - CSS = """ - Screen { background: #1e1e2e; color: #cdd6f4; } - #mode-bar { height: auto; background: #181825; padding: 0 1; border-bottom: solid #313244; } - .title-badge { padding: 0 1; background: #89b4fa; color: #11111b; text-style: bold; } - #mode-hints { color: #a6adc8; padding-left: 2; } - #search-bar-container { height: auto; background: #313244; padding: 0 1; align: left middle; } - #search-bar-container.hidden { display: none; } - #search-input { width: 1fr; } - #search-input.not-found { color: #f38ba8; border: tall #f38ba8; } - #search-status { padding: 0 1; width: auto; height: 100%; content-align: center middle; } - #search-status.not-found { color: #f38ba8; text-style: bold; } - #search-status.matched { color: #a6e3a1; text-style: bold; } - #split-container { width: 100%; height: 1fr; } - #tree-container { width: 60%; height: 100%; border-right: solid #313244; } - ParamsTree { width: 100%; height: 100%; background: #1e1e2e; } - #detail-pane { width: 40%; height: 100%; padding: 1; background: #181825; } - #detail-content { width: 100%; height: auto; } +class ParamsInspectorScreen(Screen, inherit_bindings=False): + """Textual Screen for inspecting experimaestro parameters. + + Supports both standalone CLI usage and embedding within ExperimaestroUI. + Can load parameters from a local file, pre-parsed ParamsGraph, or + on-demand via StateProvider (for remote clusters). """ + CSS_PATH = APP_TCSS_PATH + BINDINGS = [ Binding("g", "toggle_grouping", "Group Fields", show=True), Binding("p", "toggle_paths", "Paths", show=True), @@ -827,67 +822,169 @@ class ParamsInspectorApp(App): Binding("y", "copy_python", "Copy Python", show=True), Binding("J", "toggle_raw_json", "Raw JSON", key_display="J", show=True), Binding("j", "jump_primary", "Jump", show=True), - Binding("escape", "dismiss_or_quit", "Back/Quit", show=True), - Binding("q", "quit", "Quit", show=False), + Binding("escape", "dismiss_or_quit", "Back", show=True, priority=True), + Binding("q", "dismiss_or_quit", "Back", show=False, priority=True), ] - def __init__(self, params_path: Path): - """Initialize the parameters inspector application. + def __init__( + self, + params_target: Path | str | ParamsGraph | None = None, + job: Optional["BaseJob"] = None, + state_provider: Optional["StateProvider"] = None, + title: Optional[str] = None, + ): + """Initialize the parameters inspector screen. Args: - params_path: Filesystem path to the parameters JSON file. + params_target: Filesystem path or already-parsed ParamsGraph. + job: BaseJob instance for StateProvider retrieval. + state_provider: StateProvider for fetching parameters. + title: Custom title for the screen header. """ super().__init__() - self.params_path = Path(params_path).resolve() - self.graph = ParamsGraph.from_file(self.params_path) - jname = self.graph.job_path.name if self.graph.job_path else "job" - self.sub_title = f"{jname} ({self.params_path.parent.parent.name if self.graph.job_path else ''})" + self.job = job + self.state_provider = state_provider + self.custom_title = title + self.params_path: Optional[Path] = None + self.graph: Optional[ParamsGraph] = None + self._loading = False + + if isinstance(params_target, ParamsGraph): + self.graph = params_target + if self.graph.job_path: + self.params_path = self.graph.job_path + elif params_target is not None: + self.params_path = Path(params_target).resolve() + self.graph = ParamsGraph.from_file(self.params_path) + elif self.job is not None and self.state_provider is not None: + if not self.state_provider.is_remote: + p = self.state_provider.get_job_params_file(self.job) + if p: + self.params_path = p + self.graph = ParamsGraph.from_file(p) + else: + self._loading = True + else: + self._loading = True + else: + raise ValueError( + "Either params_target or (job and state_provider) must be provided" + ) + + if self.custom_title: + self.title = self.custom_title + elif self.params_path and self.graph: + jname = self.graph.job_path.name if self.graph.job_path else "job" + self.sub_title = f"{jname} ({self.params_path.parent.parent.name if self.graph.job_path else ''})" def compose(self) -> ComposeResult: - """Compose top-level layout of the application.""" + """Compose top-level layout of the screen.""" yield Header() - with Horizontal(id="mode-bar"): - yield Static(" ⚙️ Config Tree ", id="view-title", classes="title-badge") + if self._loading: yield Static( - " [g] Group · [p] Paths · [J] Raw JSON · [/] Search", - id="mode-hints", + "Syncing parameters from remote...", + id="loading", + classes="loading-message", ) - with Horizontal(id="search-bar-container", classes="hidden"): - yield Input(placeholder="Filter parameters...", id="search-input") - yield Static("", id="search-status") - with Horizontal(id="split-container"): - with Vertical(id="tree-container"): - yield ParamsTree(self.graph) - yield NodeDetailPane(self.graph) + else: + with Horizontal(id="mode-bar"): + yield Static(" ⚙️ Config Tree ", id="view-title", classes="title-badge") + yield Static( + " [g] Group · [p] Paths · [J] Raw JSON · [/] Search", + id="mode-hints", + ) + with Horizontal(id="search-bar-container", classes="hidden"): + yield Input(placeholder="Filter parameters...", id="search-input") + yield Static("", id="search-status") + with Horizontal(id="split-container"): + with Vertical(id="tree-container"): + yield ParamsTree(self.graph) + yield NodeDetailPane(self.graph) yield Footer() def on_mount(self) -> None: - """Set initial focus to the parameters tree on mount.""" - self.query_one(ParamsTree).focus() + """Set initial focus or start remote sync on mount.""" + if self._loading: + self._start_remote_sync() + else: + self.query_one(ParamsTree).focus() - def on_tree_node_highlighted(self, event: Tree.NodeHighlighted) -> None: - """Update detail inspector pane when cursor moves over a node. + @work(thread=True) + def _start_remote_sync(self) -> None: + """Fetch parameters file via StateProvider in background thread.""" + try: + if self.state_provider and self.job: + p = self.state_provider.get_job_params_file(self.job) + if p and p.is_file(): + self.params_path = p + self.graph = ParamsGraph.from_file(p) + self.app.call_from_thread(self._on_sync_success) + return + self.app.call_from_thread( + self._on_sync_failure, "Parameters file not found" + ) + except Exception as e: + self.app.call_from_thread(self._on_sync_failure, str(e)) + + def _on_sync_success(self) -> None: + """Mount inspector widgets once remote parameters have been retrieved.""" + self._loading = False + try: + loading = self.query_one("#loading", Static) + loading.remove() + except Exception: + pass + + footer = self.query_one(Footer) + mode_bar = Horizontal( + Static(" ⚙️ Config Tree ", id="view-title", classes="title-badge"), + Static( + " [g] Group · [p] Paths · [J] Raw JSON · [/] Search", + id="mode-hints", + ), + id="mode-bar", + ) + search_bar = Horizontal( + Input(placeholder="Filter parameters...", id="search-input"), + Static("", id="search-status"), + id="search-bar-container", + classes="hidden", + ) + split = Horizontal( + Vertical(ParamsTree(self.graph), id="tree-container"), + NodeDetailPane(self.graph), + id="split-container", + ) + self.mount(mode_bar, before=footer) + self.mount(search_bar, before=footer) + self.mount(split, before=footer) - Args: - event: Textual NodeHighlighted event. - """ + if not self.custom_title and self.params_path and self.graph: + jname = self.graph.job_path.name if self.graph.job_path else "job" + self.sub_title = f"{jname} ({self.params_path.parent.parent.name if self.graph.job_path else ''})" + + tree = self.query_one(ParamsTree) + tree.focus() + + def _on_sync_failure(self, error_msg: str) -> None: + """Display error message when sync fails.""" + try: + loading = self.query_one("#loading", Static) + loading.update(f"⚠️ {error_msg}\n\nPress Esc or 'q' to go back.") + except Exception: + pass + + def on_tree_node_highlighted(self, event: Tree.NodeHighlighted) -> None: + """Update detail inspector pane when cursor moves over a node.""" self.query_one(NodeDetailPane).update_detail(event.node.data) def on_tree_node_selected(self, event: Tree.NodeSelected) -> None: - """Activate selected node, jumping to primary definition if shared. - - Args: - event: Textual NodeSelected event. - """ + """Activate selected node, jumping to primary definition if shared.""" if event.node.data and event.node.data.target_object_id: self.action_jump_primary() def on_input_changed(self, event: Input.Changed) -> None: - """Filter tree view as query string changes. - - Args: - event: Textual Input.Changed event. - """ + """Filter tree view as query string changes.""" tree = self.query_one(ParamsTree) tree.set_filter(event.value) inp = self.query_one("#search-input", Input) @@ -897,34 +994,33 @@ def on_input_changed(self, event: Input.Changed) -> None: q = event.value.strip() if q: if tree.match_count == 0: - inp.add_class("not-found") - status.remove_class("matched") - status.add_class("not-found") + inp.add_class("error") + inp.remove_class("valid") + status.remove_class("valid") + status.add_class("error") status.update(" (not found)") hints.update( f" [g] Group · [p] Paths · [/] Search: '{q}' [bold red](not found)[/bold red]" ) else: - inp.remove_class("not-found") - status.remove_class("not-found") - status.add_class("matched") + inp.remove_class("error") + inp.add_class("valid") + status.remove_class("error") + status.add_class("valid") status.update(f" ({tree.match_count} matched)") hints.update( f" [g] Group · [p] Paths · [/] Search: '{q}' [green]({tree.match_count} matched)[/green]" ) else: - inp.remove_class("not-found") - status.remove_class("not-found") - status.remove_class("matched") + inp.remove_class("error") + inp.remove_class("valid") + status.remove_class("error") + status.remove_class("valid") status.update("") hints.update(" [g] Group · [p] Paths · [J] Raw JSON · [/] Search") def on_input_submitted(self, event: Input.Submitted) -> None: - """Hide search bar and return focus to tree upon query submission. - - Args: - event: Textual Input.Submitted event. - """ + """Hide search bar and return focus to tree upon query submission.""" self.query_one("#search-bar-container").add_class("hidden") self.query_one(ParamsTree).focus() @@ -1042,22 +1138,97 @@ def action_jump_primary(self) -> None: self.notify(f"Jumped to primary definition #{tid}") def action_dismiss_or_quit(self) -> None: - """Dismiss the search bar if visible, or exit the application.""" - box = self.query_one("#search-bar-container") - if "hidden" not in box.classes: - box.add_class("hidden") - inp = self.query_one("#search-input", Input) - inp.value = "" - inp.remove_class("not-found") - status = self.query_one("#search-status", Static) - status.remove_class("not-found") - status.remove_class("matched") - status.update("") - self.query_one("#mode-hints", Static).update( - " [g] Group · [p] Paths · [J] Raw JSON · [/] Search" - ) - tree = self.query_one(ParamsTree) - tree.set_filter("") - tree.focus() - return - self.exit() + """Dismiss the search bar if visible, or exit the application/screen.""" + if not self._loading: + try: + box = self.query_one("#search-bar-container") + if "hidden" not in box.classes: + box.add_class("hidden") + inp = self.query_one("#search-input", Input) + inp.value = "" + inp.remove_class("error") + inp.remove_class("valid") + status = self.query_one("#search-status", Static) + status.remove_class("error") + status.remove_class("valid") + status.update("") + self.query_one("#mode-hints", Static).update( + " [g] Group · [p] Paths · [J] Raw JSON · [/] Search" + ) + tree = self.query_one(ParamsTree) + tree.set_filter("") + tree.focus() + return + except Exception: + pass + + if len(self.app.screen_stack) > 1: + self.dismiss() + else: + self.app.exit() + + +class ParamsInspectorApp(App): + """Standalone Textual App for inspecting experimaestro params.json. + + Features interactive tree navigation, categorization, path resolution, + fuzzy search filtering, Python snippet copying, and raw JSON inspection. + """ + + TITLE = "Experimaestro Parameters Inspector" + CSS_PATH = APP_TCSS_PATH + + def __init__(self, params_path: Path): + """Initialize the parameters inspector application. + + Args: + params_path: Filesystem path to the parameters JSON file. + """ + super().__init__() + self.params_path = Path(params_path).resolve() + self.inspector_screen = ParamsInspectorScreen(self.params_path) + + def get_default_screen(self) -> Screen: + """Return the inspector screen as the default screen.""" + return self.inspector_screen + + @property + def graph(self) -> Optional[ParamsGraph]: + """Accessor for the underlying ParamsGraph.""" + return self.inspector_screen.graph + + def action_expand_all(self) -> None: + """Forward expand_all to inspector screen.""" + self.inspector_screen.action_expand_all() + + def action_collapse_all(self) -> None: + """Forward collapse_all to inspector screen.""" + self.inspector_screen.action_collapse_all() + + def action_toggle_grouping(self) -> None: + """Forward toggle_grouping to inspector screen.""" + self.inspector_screen.action_toggle_grouping() + + def action_toggle_paths(self) -> None: + """Forward toggle_paths to inspector screen.""" + self.inspector_screen.action_toggle_paths() + + def action_toggle_raw_json(self) -> None: + """Forward toggle_raw_json to inspector screen.""" + self.inspector_screen.action_toggle_raw_json() + + def action_toggle_search(self) -> None: + """Forward toggle_search to inspector screen.""" + self.inspector_screen.action_toggle_search() + + def action_copy_value(self) -> None: + """Forward copy_value to inspector screen.""" + self.inspector_screen.action_copy_value() + + def action_copy_python(self) -> None: + """Forward copy_python to inspector screen.""" + self.inspector_screen.action_copy_python() + + def action_jump_primary(self) -> None: + """Forward jump_primary to inspector screen.""" + self.inspector_screen.action_jump_primary() diff --git a/src/experimaestro/tui/__init__.py b/src/experimaestro/tui/__init__.py index e7baf1bc..3a3fcd13 100644 --- a/src/experimaestro/tui/__init__.py +++ b/src/experimaestro/tui/__init__.py @@ -1,8 +1,11 @@ """Textual-based TUI for monitoring experiments""" +from pathlib import Path from .app import ExperimaestroUI # Backward compatibility alias ExperimentTUI = ExperimaestroUI -__all__ = ["ExperimaestroUI", "ExperimentTUI"] +APP_TCSS_PATH = Path(__file__).parent / "app.tcss" + +__all__ = ["ExperimaestroUI", "ExperimentTUI", "APP_TCSS_PATH"] diff --git a/src/experimaestro/tui/app.py b/src/experimaestro/tui/app.py index 44405142..a48b1aac 100644 --- a/src/experimaestro/tui/app.py +++ b/src/experimaestro/tui/app.py @@ -40,6 +40,8 @@ JobHighlighted, ViewJobLogs, ViewJobLogsRequest, + InspectJobParams, + InspectJobParamsRequest, DeleteJobRequest, DeleteExperimentRequest, KillJobRequest, @@ -82,6 +84,7 @@ class ExperimaestroUI(App): Binding("?", "show_help", "Help"), Binding("escape", "go_back", "Back", show=False), Binding("l", "view_logs", "Logs", show=False), + Binding("i", "inspect_params", "Inspect Params", show=False), Binding("w", "view_workspace_log", "Workspace Log", show=False), Binding("j", "focus_jobs", "Jobs", show=False), Binding("s", "focus_services", "Services", show=False), @@ -697,6 +700,13 @@ def action_view_logs(self) -> None: job_detail_view = self.query_one(JobDetailView) job_detail_view.action_view_logs() + def action_inspect_params(self) -> None: + """Inspect parameters for the current job (if job detail is visible)""" + job_detail_container = self.query_one("#job-detail-container") + if not job_detail_container.has_class("hidden"): + job_detail_view = self.query_one(JobDetailView) + job_detail_view.action_inspect_params() + def action_view_workspace_log(self) -> None: """View workspace-level experimaestro log for current run""" if not self.state_provider: @@ -787,6 +797,43 @@ def on_view_job_logs_request(self, message: ViewJobLogsRequest) -> None: return self.post_message(ViewJobLogs(str(job.path), job.task_id, job.state)) + def on_inspect_job_params_request(self, message: InspectJobParamsRequest) -> None: + """Handle params inspection request from jobs table or tree""" + job = self.state_provider.get_job(message.task_id, message.job_id) + if not job: + self.notify("Cannot find job", severity="warning") + return + self.post_message(InspectJobParams(job)) + + def on_inspect_job_params(self, message: InspectJobParams) -> None: + """Handle request to inspect job parameters - push ParamsInspectorScreen""" + from experimaestro.tools.inspect_params import ParamsInspectorScreen + + job = message.job + if not self.state_provider.is_remote: + params_file = self.state_provider.get_job_params_file(job) + if not params_file or not params_file.is_file(): + self.notify( + f"Parameters file not found for {job.identifier[:8]}", + severity="warning", + ) + return + self.push_screen( + ParamsInspectorScreen( + params_target=params_file, + title=f"Job {job.identifier[:8]}", + ) + ) + else: + self.push_screen( + ParamsInspectorScreen( + params_target=None, + job=job, + state_provider=self.state_provider, + title=f"Job {job.identifier[:8]}", + ) + ) + def on_delete_job_request(self, message: DeleteJobRequest) -> None: """Handle job deletion request""" job = self.state_provider.get_job(message.task_id, message.job_id) diff --git a/src/experimaestro/tui/app.tcss b/src/experimaestro/tui/app.tcss index b34bec12..c092aa0c 100644 --- a/src/experimaestro/tui/app.tcss +++ b/src/experimaestro/tui/app.tcss @@ -541,3 +541,95 @@ StrayJobsPanel Static, OrphanJobsPanel Static { height: auto; } + +/* Parameters Inspector styles */ +ParamsInspectorScreen { + background: $surface; + color: $text; +} + +#mode-bar { + height: auto; + background: $boost; + padding: 0 1; + border-bottom: solid $primary; +} + +.title-badge { + padding: 0 1; + background: $primary; + color: $text; + text-style: bold; +} + +#mode-hints { + color: $text-muted; + text-style: italic; + padding-left: 2; +} + +#search-bar-container { + height: auto; + background: $boost; + padding: 0 1; + align: left middle; +} + +#search-status { + padding: 0 1; + width: auto; + height: 100%; + content-align: center middle; +} + +#search-status.error { + color: $error; + text-style: bold; +} + +#search-status.valid { + color: $success; + text-style: bold; +} + +#split-container { + width: 100%; + height: 1fr; +} + +#tree-container { + width: 60%; + height: 100%; + border-right: solid $primary; +} + +ParamsTree { + width: 100%; + height: 100%; + background: $surface; +} + +#detail-pane { + width: 40%; + height: 100%; + padding: 1; + background: $surface; +} + +#detail-content { + width: 100%; + height: auto; +} + +.loading-message { + content-align: center middle; + height: 100%; + text-style: italic; + color: $text-muted; +} + +.error-message { + content-align: center middle; + height: 100%; + color: $error; +} diff --git a/src/experimaestro/tui/dialogs.py b/src/experimaestro/tui/dialogs.py index 2370eb66..b0f77ed5 100644 --- a/src/experimaestro/tui/dialogs.py +++ b/src/experimaestro/tui/dialogs.py @@ -137,6 +137,7 @@ def compose(self) -> ComposeResult: [bold cyan]Jobs[/bold cyan] l View job logs + i Inspect job parameters Ctrl+d Delete job Ctrl+k Kill running job / Open search filter diff --git a/src/experimaestro/tui/messages.py b/src/experimaestro/tui/messages.py index 4c4b26f9..8a51e8c9 100644 --- a/src/experimaestro/tui/messages.py +++ b/src/experimaestro/tui/messages.py @@ -4,7 +4,7 @@ from textual.message import Message if TYPE_CHECKING: - from experimaestro.scheduler.interfaces import JobState + from experimaestro.scheduler.interfaces import BaseJob, JobState class ExperimentSelected(Message): @@ -60,6 +60,24 @@ def __init__(self, job_id: str, task_id: str, experiment_id: str) -> None: self.experiment_id = experiment_id +class InspectJobParams(Message): + """Message sent when user wants to inspect job parameters""" + + def __init__(self, job: "BaseJob") -> None: + super().__init__() + self.job = job + + +class InspectJobParamsRequest(Message): + """Message sent when user requests to inspect parameters from jobs table or tree""" + + def __init__(self, job_id: str, task_id: str, experiment_id: str) -> None: + super().__init__() + self.job_id = job_id + self.task_id = task_id + self.experiment_id = experiment_id + + class DeleteJobRequest(Message): """Message sent when user requests to delete a job""" diff --git a/src/experimaestro/tui/widgets/jobs.py b/src/experimaestro/tui/widgets/jobs.py index 7612776d..fd1b3071 100644 --- a/src/experimaestro/tui/widgets/jobs.py +++ b/src/experimaestro/tui/widgets/jobs.py @@ -23,6 +23,8 @@ JobHighlighted, ViewJobLogs, ViewJobLogsRequest, + InspectJobParams, + InspectJobParamsRequest, DeleteJobRequest, KillJobRequest, FilterChanged, @@ -178,6 +180,7 @@ class JobDetailView(Widget): BINDINGS = [ Binding("l", "view_logs", "View Logs", priority=True), + Binding("i", "inspect_params", "Inspect Params", priority=True), ] def __init__(self, state_provider: StateProvider) -> None: @@ -226,6 +229,11 @@ def action_view_logs(self) -> None: ) ) + def action_inspect_params(self) -> None: + """Inspect job parameters with TUI inspector""" + if self.job_data: + self.post_message(InspectJobParams(self.job_data)) + def _get_process_info(self, job) -> str: """Get process information for a job using the state provider""" pinfo = self.state_provider.get_process_info(job) @@ -503,21 +511,29 @@ def format_time(ts): progress_text = "-" self.query_one("#job-progress-label", Label).update(progress_text) - # Log files hint - log files are named after the last part of the task ID + # Actions hint (logs and parameters) job_path = job.path task_id = job.task_id if job_path and task_id: - # Extract the last component of the task ID (e.g., "evaluate" from "mnist_xp.learn.evaluate") - task_name = task_id.split(".")[-1] - stdout_path = job_path / f"{task_name}.out" - stderr_path = job_path / f"{task_name}.err" - logs_exist = stdout_path.exists() or stderr_path.exists() - if logs_exist: - self.query_one("#job-logs-hint", Label).update( - "[bold cyan]Press 'l' to view logs[/bold cyan]" - ) + if self.state_provider.is_remote: + logs_exist = bool(job.state and job.state != JobState.SCHEDULED) + params_exist = self.state_provider.has_job_params(job) + else: + task_name = task_id.split(".")[-1] + stdout_path = job_path / f"{task_name}.out" + stderr_path = job_path / f"{task_name}.err" + logs_exist = stdout_path.exists() or stderr_path.exists() + params_exist = self.state_provider.has_job_params(job) + + if logs_exist and params_exist: + hint = "[bold cyan]Press 'l' to view logs, 'i' to inspect params[/bold cyan]" + elif params_exist: + hint = "[bold cyan]Press 'i' to inspect params[/bold cyan] [dim](no logs yet)[/dim]" + elif logs_exist: + hint = "[bold cyan]Press 'l' to view logs[/bold cyan]" else: - self.query_one("#job-logs-hint", Label).update("(no log files found)") + hint = "(no log or param files found)" + self.query_one("#job-logs-hint", Label).update(hint) else: self.query_one("#job-logs-hint", Label).update("") @@ -572,6 +588,7 @@ class JobsTreeView(Widget): Binding("E", "collapse_all", "Collapse All", show=False), Binding("enter", "select_node", "Select", show=False), Binding("l", "view_logs", "Logs", key_display="l"), + Binding("i", "inspect_params", "Inspect", key_display="i"), Binding("ctrl+d", "delete_job", "Delete", show=False), Binding("ctrl+k", "kill_job", "Kill", show=False), ] @@ -1098,6 +1115,15 @@ def action_view_logs(self) -> None: ViewJobLogsRequest(data.job_id, data.task_id, self.current_experiment) ) + def action_inspect_params(self) -> None: + data = self._get_selected_job_data() + if data and self.current_experiment: + self.post_message( + InspectJobParamsRequest( + data.job_id, data.task_id, self.current_experiment + ) + ) + def action_delete_job(self) -> None: data = self._get_selected_job_data() if data and self.current_experiment: @@ -1128,6 +1154,7 @@ class JobsTable(Vertical): Binding("ctrl+d", "delete_job", "Delete", show=False), Binding("ctrl+k", "kill_job", "Kill", show=False), Binding("l", "view_logs", "Logs", key_display="l"), + Binding("i", "inspect_params", "Inspect", key_display="i"), Binding("f", "copy_path", "Copy Path", show=False), Binding("/", "toggle_search", "Search"), Binding("c", "clear_filter", "Clear", show=False), @@ -1360,6 +1387,18 @@ def action_view_logs(self) -> None: ViewJobLogsRequest(job_id, task_id, self.current_experiment) ) + def action_inspect_params(self) -> None: + """Request to inspect params for the selected job""" + selected = self._get_selected_job_info() + if selected is None: + return + + job_id, task_id = selected + if task_id: + self.post_message( + InspectJobParamsRequest(job_id, task_id, self.current_experiment) + ) + def _get_selected_job_info(self) -> tuple[str, str] | None: """Get (job_id, task_id) for the currently selected job in either view. From 9ef71ea95de457fa1ac886fd05e920a51a5c99d4 Mon Sep 17 00:00:00 2001 From: Victor Morand Date: Tue, 20 Oct 2026 10:19:08 +0200 Subject: [PATCH 5/7] refactor: dag can load from remote workspaces --- .../tests/test_grid_validation.py | 5 +- .../tests/test_inspect_params.py | 56 ++++++ src/experimaestro/tests/test_objects.py | 1 - src/experimaestro/tests/test_serializers.py | 3 +- src/experimaestro/tests/test_xpm_dag.py | 147 +++++++++++++++ .../tools/inspect_params/discovery.py | 8 +- .../tools/inspect_params/models.py | 165 ++++++++++++----- src/experimaestro/tools/xpm_dag/xpm_dag.py | 167 ++++++++++++++---- 8 files changed, 473 insertions(+), 79 deletions(-) diff --git a/src/experimaestro/tests/test_grid_validation.py b/src/experimaestro/tests/test_grid_validation.py index 5f6335fe..76a29a81 100644 --- a/src/experimaestro/tests/test_grid_validation.py +++ b/src/experimaestro/tests/test_grid_validation.py @@ -177,5 +177,8 @@ def test_validate_attrs_warns_on_unrecognized_keys(caplog): cfg = validate_attrs(MainConfig, data) assert cfg.id == "test" - assert any("WARNING: Unrecognized configuration keys" in record.message for record in caplog.records) + assert any( + "WARNING: Unrecognized configuration keys" in record.message + for record in caplog.records + ) assert any("unexpected_key_xyz" in record.message for record in caplog.records) diff --git a/src/experimaestro/tests/test_inspect_params.py b/src/experimaestro/tests/test_inspect_params.py index 8eb04697..59bb0b32 100644 --- a/src/experimaestro/tests/test_inspect_params.py +++ b/src/experimaestro/tests/test_inspect_params.py @@ -872,3 +872,59 @@ def test_cli_inspect_help(): assert res.exit_code == 0 assert "Interactive TUI Parameters Inspector" in res.output assert "--workspace" in res.output + + +def test_params_graph_objects_jsonl_and_value_resolution(tmp_path): + """Verify ParamsGraph and find_params_file support objects.jsonl and resolve_serialized_value.""" + from experimaestro.tools.inspect_params import resolve_serialized_value + + objs = [ + { + "id": 10, + "module": "pkg.models", + "type": "Encoder", + "identifier": "enc_hash", + "fields": { + "dim": 512, + "mode": {"type": "enum", "enum": "Mode", "value": "TRAIN"}, + "tags_set": {"type": "set", "items": ["a", "b"]}, + }, + }, + { + "id": 20, + "module": "pkg.tasks", + "type": "LearnTask", + "identifier": "learn_hash", + "fields": {"model": {"type": "python", "value": 10}}, + }, + { + "id": 30, + "module": "pkg.models", + "type": "CheckpointLoader", + "identifier": "ckpt_hash", + "task": 20, + "fields": {"path": {"type": "path", "value": "/tmp/ckpt"}}, + }, + ] + + jsonl_path = tmp_path / "objects.jsonl" + jsonl_path.write_text( + "\n".join(json.dumps(o) for o in objs) + "\n", encoding="utf-8" + ) + + assert find_params_file(str(tmp_path)) == jsonl_path + graph = ParamsGraph.from_file(tmp_path, load_docstrings=False) + + assert len(graph.objects) == 3 + assert 20 in graph.task_object_ids + assert graph.objects[20].is_task is True + assert graph.objects[30].task_id == 20 + + resolved_fields = resolve_serialized_value(graph.objects[10].fields, graph.objects) + assert resolved_fields["dim"] == 512 + assert resolved_fields["mode"] == "TRAIN" + assert resolved_fields["tags_set"] == ["a", "b"] + assert ( + resolve_serialized_value(graph.objects[20].fields["model"], graph.objects) + == graph.objects[10] + ) diff --git a/src/experimaestro/tests/test_objects.py b/src/experimaestro/tests/test_objects.py index c1672cd8..27c04bf0 100644 --- a/src/experimaestro/tests/test_objects.py +++ b/src/experimaestro/tests/test_objects.py @@ -438,4 +438,3 @@ def test_field_names_and_load_objects_missing_defaults(): # In-place mutations on default containers work as expected inst.items.append("test") assert inst.items == ["test"] - diff --git a/src/experimaestro/tests/test_serializers.py b/src/experimaestro/tests/test_serializers.py index 712a97cf..ac9e3ca7 100644 --- a/src/experimaestro/tests/test_serializers.py +++ b/src/experimaestro/tests/test_serializers.py @@ -257,8 +257,7 @@ def test_taskstub_serialization_roundtrip(): [reloaded_config] = from_state_dict(data2, partial_loading=False) assert isinstance(reloaded_config.__xpm__.task, TaskStub) assert ( - reloaded_config.__xpm__.task.identifier - == loaded_config.__xpm__.task.identifier + reloaded_config.__xpm__.task.identifier == loaded_config.__xpm__.task.identifier ) assert reloaded_config.__xpm__.task.typename == loaded_config.__xpm__.task.typename diff --git a/src/experimaestro/tests/test_xpm_dag.py b/src/experimaestro/tests/test_xpm_dag.py index 493d27a2..b59bf806 100644 --- a/src/experimaestro/tests/test_xpm_dag.py +++ b/src/experimaestro/tests/test_xpm_dag.py @@ -558,3 +558,150 @@ def mock_fetch(remote_ws, exp_id, r_id): assert resolved_id == "20260101_120000" assert run_d == fake_run_dir assert called == [("cluster_ws", "my_remote_exp", "20260101_120000")] + + +def test_load_run_graph_without_importing_remote_modules(tmp_path): + """Verify load_run_graph builds the DAG from JSONL without importing remote modules or files.""" + from experimaestro import Action, Config, Task + from experimaestro.tools.xpm_dag.xpm_dag import load_run_graph + + run_dir = tmp_path / "remote_run" + run_dir.mkdir() + + cfg_id = "a" * 64 + task_id = "b" * 64 + out_cfg_id = "c" * 64 + eval_task_id = "d" * 64 + action_id = "e" * 64 + + objects_data = [ + { + "id": 1, + "module": "non_existent_remote_pkg.models", + "type": "RemoteEncoderConfig", + "typename": "non_existent_remote_pkg.models.RemoteEncoderConfig", + "identifier": cfg_id, + "file": "/lustre/fsn1/projects/rech/cdt/uqo57sk/code/cross-encoders/src/tests.py", + "fields": { + "hidden_dim": 768, + "lr": 0.0001, + "mode": { + "type": "enum", + "module": "non_existent_remote_pkg.enums", + "enum": "Mode", + "value": "FAST", + }, + "data_path": { + "type": "path", + "base": "workspace", + "value": "data/corpus", + }, + "generated_listener_path": { + "type": "path", + "value": f"/lustre/ws/jobs/non_existent_remote_pkg.tasks.learn/{task_id}/out/listener", + }, + }, + }, + { + "id": 2, + "module": "non_existent_remote_pkg.tasks", + "type": "Learn", + "typename": "non_existent_remote_pkg.tasks.Learn", + "identifier": task_id, + "file": "/lustre/fsn1/projects/rech/cdt/uqo57sk/code/cross-encoders/src/training_utils.py", + "fields": { + "encoder": {"type": "python", "value": 1}, + }, + }, + { + "id": 3, + "module": "non_existent_remote_pkg.models", + "type": "ParameterLoader", + "typename": "non_existent_remote_pkg.models.ParameterLoader", + "identifier": out_cfg_id, + "task": 2, + "fields": {}, + }, + { + "id": 4, + "module": "tests", + "type": "Evaluate", + "typename": "tests.Evaluate", + "identifier": eval_task_id, + "file": "/lustre/fsn1/projects/rech/cdt/uqo57sk/code/cross-encoders/src/tests.py", + "init-tasks": [3], + "fields": { + "model": {"type": "python", "value": 1}, + }, + }, + { + "id": 5, + "module": "non_existent_remote_pkg.actions", + "type": "ExportHuggingFace", + "typename": "non_existent_remote_pkg.actions.ExportHuggingFace", + "identifier": action_id, + "fields": { + "task": {"type": "python", "value": 2}, + }, + }, + ] + + with (run_dir / "objects.jsonl").open("w") as f: + for obj in objects_data: + f.write(json.dumps(obj) + "\n") + + jobs_data = [ + { + "job_id": task_id, + "task_id": "non_existent_remote_pkg.tasks.learn", + "tags": {"lr": 0.0001, "hidden_dim": 768}, + }, + { + "job_id": eval_task_id, + "task_id": "tests.evaluate", + "tags": {"lr": 0.0001, "hidden_dim": 768}, + }, + ] + with (run_dir / "jobs.jsonl").open("w") as f: + for j in jobs_data: + f.write(json.dumps(j) + "\n") + + status_data = { + "job_states": {task_id: "done", eval_task_id: "running"}, + "actions": {action_id: {"description": "Export model"}}, + } + (run_dir / "status.json").write_text(json.dumps(status_data)) + + res = load_run_graph(run_dir) + assert res is not None + full_jobs, dependencies, job_tags_dict = res + + assert set(full_jobs.keys()) == { + cfg_id, + task_id, + out_cfg_id, + eval_task_id, + action_id, + } + assert isinstance(full_jobs[cfg_id], Config) + assert not isinstance(full_jobs[cfg_id], (Task, Action)) + assert isinstance(full_jobs[task_id], Task) + assert isinstance(full_jobs[eval_task_id], Task) + assert isinstance(full_jobs[action_id], Action) + + assert type(full_jobs[cfg_id]).__name__ == "RemoteEncoderConfig" + assert type(full_jobs[task_id]).__name__ == "Learn" + assert type(full_jobs[action_id]).__name__ == "ExportHuggingFace" + + assert cfg_id in dependencies[task_id] + assert task_id not in dependencies.get(cfg_id, []) + assert task_id in dependencies[out_cfg_id] + assert out_cfg_id in dependencies[eval_task_id] + assert cfg_id in dependencies[eval_task_id] + assert task_id in dependencies[action_id] + + assert job_tags_dict[cfg_id] == {"hidden_dim": 768, "lr": 0.0001} + assert job_tags_dict[task_id] == {"hidden_dim": 768, "lr": 0.0001} + + dag_json = build_dag_json(full_jobs, dependencies, job_tags_dict, run_dir=run_dir) + assert len(dag_json["nodes"]) == 5 diff --git a/src/experimaestro/tools/inspect_params/discovery.py b/src/experimaestro/tools/inspect_params/discovery.py index 066427bf..92ff67ed 100644 --- a/src/experimaestro/tools/inspect_params/discovery.py +++ b/src/experimaestro/tools/inspect_params/discovery.py @@ -184,11 +184,15 @@ def _find_params_in_path(p: Path) -> Optional[Path]: if p.is_file(): return p.resolve() if p.is_dir(): - for name in ("params.json", "experimaestro.json"): + for name in ("params.json", "experimaestro.json", "objects.jsonl"): cand = p / name if cand.is_file(): return cand.resolve() - m = list(p.glob("**/params.json")) + list(p.glob("**/experimaestro.json")) + m = ( + list(p.glob("**/params.json")) + + list(p.glob("**/experimaestro.json")) + + list(p.glob("**/objects.jsonl")) + ) if m: return m[0].resolve() return None diff --git a/src/experimaestro/tools/inspect_params/models.py b/src/experimaestro/tools/inspect_params/models.py index c3a9e4e3..00c3ecaf 100644 --- a/src/experimaestro/tools/inspect_params/models.py +++ b/src/experimaestro/tools/inspect_params/models.py @@ -12,11 +12,55 @@ import json import logging from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union + +from experimaestro.core.identifier import Identifier logger = logging.getLogger("experimaestro") +def normalize_identifier(raw_ident: Any) -> str: + """Normalize a serialized identifier (string hash or Identifier state dict).""" + if isinstance(raw_ident, dict): + return str(Identifier.from_state_dict(raw_ident)) + return str(raw_ident) if raw_ident is not None else "" + + +def resolve_serialized_value(value: Any, objects: Mapping[int, Any]) -> Any: + """Resolve a serialized JSON field value against a mapping of loaded objects.""" + if isinstance(value, list): + return [resolve_serialized_value(x, objects) for x in value] + + if isinstance(value, dict): + if "type" not in value: + return {k: resolve_serialized_value(v, objects) for k, v in value.items()} + + vtype = value["type"] + if vtype == "python": + ref_id = int(value["value"]) + if ref_id not in objects: + logger.warning( + "Referenced object id %s not found in loaded objects", ref_id + ) + return None + return objects[ref_id] + + if vtype in ("path", "path.serialized"): + return Path(value["value"]) + + if vtype == "set": + return [ + resolve_serialized_value(x, objects) for x in value.get("items", []) + ] + + if vtype == "enum": + return value.get("value") + + return value.get("value") + + return value + + @dataclass class ResolvedPath: """Representation of a path reference resolved against job or workspace directories. @@ -104,19 +148,59 @@ class ObjectNode: incoming_refs: List[Tuple[int, str]] = field(default_factory=list) outgoing_refs: List[Tuple[str, int]] = field(default_factory=list) docstring: Optional[str] = None + task_id: Optional[int] = None + + @classmethod + def from_definition( + cls, o: Dict[str, Any], *, load_docstring: bool = True + ) -> ObjectNode: + """Create an ObjectNode from a raw serialized definition dictionary.""" + oid = int(o["id"]) + mod, typ = o.get("module", ""), o.get("type", "") + qname = o.get("typename") or f"{mod}.{typ}" if mod else typ or str(oid) + ident = normalize_identifier(o.get("identifier", "")) + inits = [int(x) for x in o.get("init-tasks", [])] + t_ptr = o.get("task") + task_id = int(t_ptr) if t_ptr is not None else None + is_task = (task_id is not None and task_id == oid) or bool(inits) + + doc = None + if load_docstring and mod and typ: + try: + cls_obj = getattr(importlib.import_module(mod), typ, None) + doc = cls_obj.__doc__.strip() if cls_obj and cls_obj.__doc__ else None + except Exception: + pass + + raw_fields = o.get("fields", {}) + return cls( + oid, + mod, + typ, + qname, + ident, + is_task, + False, + inits, + raw_fields if isinstance(raw_fields, dict) else {}, + docstring=doc, + task_id=task_id, + ) class ParamsGraph: """Parses, indexes, and queries the object dependency graph of an Experimaestro job. - Supports both single-job dictionaries (`params.json`) and multi-task list - payloads (such as Hugging Face Hub `experimaestro.json` exports). + Supports single-job dictionaries (`params.json`), multi-task list payloads + (such as Hugging Face Hub `experimaestro.json` exports), and streaming + experiment records (`objects.jsonl`). Attributes: raw_data: Original deserialized JSON data. job_path: Path to the directory containing the parameters file. workspace_path: Resolved path to the enclosing workspace directory. objects: Mapping of numeric object IDs to ObjectNode instances. + task_object_ids: Set of object IDs referenced via `task` or `init-tasks`. root_task_id: Object ID of the primary root task, if determined. version: Schema version number. experimaestro_version: Experimaestro version string. @@ -130,13 +214,16 @@ def __init__( data: Union[Dict[str, Any], List[Dict[str, Any]]], job_path: Optional[Path] = None, workspace_path: Optional[Path] = None, + *, + load_docstrings: bool = True, ): """Initialize the graph from a dictionary or list of raw object specifications. Args: - data: Deserialized params.json dict or experimaestro.json list. + data: Deserialized params.json dict or experimaestro.json / objects.jsonl list. job_path: Optional filesystem path to the enclosing job directory. workspace_path: Optional workspace path (auto-discovered if omitted). + load_docstrings: Whether to import modules to extract class docstrings. """ self.raw_data = data self.job_path = job_path.resolve() if job_path else None @@ -161,8 +248,9 @@ def __init__( else self._find_ws(self.job_path) ) self.objects: Dict[int, ObjectNode] = {} + self.task_object_ids: Set[int] = set() self.root_task_id: Optional[int] = None - self._parse(raw_objs) + self._parse(raw_objs, load_docstrings=load_docstrings) @staticmethod def _find_ws(path: Optional[Path]) -> Optional[Path]: @@ -175,11 +263,11 @@ def _find_ws(path: Optional[Path]) -> Optional[Path]: return None @classmethod - def from_file(cls, path: Path) -> ParamsGraph: + def from_file(cls, path: Path, *, load_docstrings: bool = True) -> ParamsGraph: """Load a parameter graph from a file or directory. - Accepts direct paths to `params.json` or `experimaestro.json`, or a directory - containing either file. + Accepts direct paths to `params.json`, `experimaestro.json`, or `objects.jsonl`, + or a directory containing any of these files. """ p = path.resolve() if p.is_dir(): @@ -187,53 +275,44 @@ def from_file(cls, path: Path) -> ParamsGraph: f = p / "params.json" elif (p / "experimaestro.json").is_file(): f = p / "experimaestro.json" + elif (p / "objects.jsonl").is_file(): + f = p / "objects.jsonl" else: f = p / "params.json" else: f = p if not f.exists(): raise FileNotFoundError(f"params.json not found at: {f}") - return cls(json.loads(f.read_text(encoding="utf-8")), job_path=f.parent) - def _parse(self, raw_objs: List[Dict[str, Any]]) -> None: + text = f.read_text(encoding="utf-8") + if f.suffix == ".jsonl" or f.name == "objects.jsonl": + payload: Union[Dict[str, Any], List[Dict[str, Any]]] = [ + json.loads(line) for line in text.splitlines() if line.strip() + ] + else: + payload = json.loads(text) + return cls(payload, job_path=f.parent, load_docstrings=load_docstrings) + + def _parse( + self, raw_objs: List[Dict[str, Any]], *, load_docstrings: bool = True + ) -> None: """Construct ObjectNode instances and resolve reference graphs.""" target_hash = self.job_path.name if self.job_path else None cand_root = None for o in raw_objs: - oid = int(o["id"]) - mod, typ = o.get("module", ""), o.get("type", "") - qname = o.get("typename") or f"{mod}.{typ}" if mod else typ or str(oid) - ident = o.get("identifier", "") - inits = [int(x) for x in o.get("init-tasks", [])] - t_ptr = o.get("task") - is_task = (t_ptr is not None and int(t_ptr) == oid) or bool(inits) - - doc = None - if mod and typ: - try: - cls_obj = getattr(importlib.import_module(mod), typ, None) - doc = ( - cls_obj.__doc__.strip() if cls_obj and cls_obj.__doc__ else None - ) - except Exception: - pass - - node = ObjectNode( - oid, - mod, - typ, - qname, - ident, - is_task, - False, - inits, - o.get("fields", {}), - docstring=doc, - ) - self.objects[oid] = node - if target_hash and ident == target_hash: - cand_root = oid + node = ObjectNode.from_definition(o, load_docstring=load_docstrings) + self.objects[node.id] = node + if node.task_id is not None: + self.task_object_ids.add(node.task_id) + for it in node.init_tasks: + self.task_object_ids.add(it) + if target_hash and node.identifier == target_hash: + cand_root = node.id + + for tid in self.task_object_ids: + if tid in self.objects: + self.objects[tid].is_task = True self.root_task_id = ( cand_root if cand_root else (int(raw_objs[-1]["id"]) if raw_objs else None) diff --git a/src/experimaestro/tools/xpm_dag/xpm_dag.py b/src/experimaestro/tools/xpm_dag/xpm_dag.py index 0ea5a82b..d4f8c012 100644 --- a/src/experimaestro/tools/xpm_dag/xpm_dag.py +++ b/src/experimaestro/tools/xpm_dag/xpm_dag.py @@ -14,13 +14,17 @@ import logging from pathlib import Path import re +import sys from typing import Any import networkx as nx -from experimaestro import tags -from experimaestro.core.objects.config import ConfigInformation +from experimaestro import Action, Config, Task, tags from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider from experimaestro.settings import find_workspace, get_workspace +from experimaestro.tools.inspect_params.models import ( + ParamsGraph, + resolve_serialized_value, +) # Default layout sizing constants BOX_W = 25.0 @@ -172,6 +176,102 @@ def extract_config_tags( return raw_params +class _LightweightParam: + """Minimal parameter descriptor carrying the argument name.""" + + __slots__ = ("name",) + + def __init__(self, name: str): + self.name = name + + +class _LightweightXPMInfo: + """Lightweight metadata container matching ConfigInformation's DAG interface.""" + + __slots__ = ("identifier", "task", "init_tasks", "dependencies", "values") + + def __init__(self, identifier: str): + self.identifier = identifier + self.task: Any = None + self.init_tasks: list[Any] = [] + self.dependencies: list[Any] = [] + self.values: dict[str, Any] = {} + + def xpmvalues(self): + for k, v in self.values.items(): + yield _LightweightParam(k), v + + def tags(self) -> dict[str, Any]: + return {} + + +def _load_lightweight_objects( + graph: ParamsGraph, + job_ids: set[str], + action_ids: set[str], +) -> dict[int, Any]: + """Construct lightweight Config, Task, and Action nodes from an indexed ParamsGraph. + + Avoids importing external Python modules or executing remote file paths while + preserving class names, isinstance(node, (Task, Action, Config)) semantics, + and __xpm__ dependency/parameter traversal. + """ + cls_cache: dict[tuple[type, str, str], type] = {} + objects: dict[int, Any] = {} + + for obj_id, obj_node in graph.objects.items(): + ident = obj_node.identifier + module_name = obj_node.module or "_xpm_dag_" + type_name = obj_node.type_name or "Unknown" + + if ident and ident in action_ids: + base_cls: type = Action + elif (ident and ident in job_ids) or obj_node.is_task: + base_cls = Task + else: + base_cls = Config + # Fallback to already-imported module in sys.modules if present + loaded_mod = sys.modules.get(module_name) + if loaded_mod is not None: + real_cls = getattr(loaded_mod, type_name, None) + if isinstance(real_cls, type): + if issubclass(real_cls, Action): + base_cls = Action + elif issubclass(real_cls, Task): + base_cls = Task + + cls_key = (base_cls, module_name, type_name) + cls = cls_cache.get(cls_key) + if cls is None: + cls = type(type_name, (base_cls,), {"__module__": module_name}) + cls_cache[cls_key] = cls + + node = object.__new__(cls) + object.__setattr__(node, "__xpm__", _LightweightXPMInfo(ident)) + objects[obj_id] = node + + for obj_id, obj_node in graph.objects.items(): + node = objects[obj_id] + xpm: _LightweightXPMInfo = node.__xpm__ + + if ( + obj_node.task_id is not None + and obj_node.task_id in objects + and obj_node.task_id != obj_id + ): + xpm.task = objects[obj_node.task_id] + + if obj_node.init_tasks: + xpm.init_tasks = [ + objects[tid] for tid in obj_node.init_tasks if tid in objects + ] + + for fname, fval in obj_node.fields.items(): + xpm.values[fname] = resolve_serialized_value(fval, objects) + + return objects + + def load_run_graph( run_dir: Path, ) -> tuple[dict, dict[str, list[str]], dict[str, dict]] | None: @@ -181,30 +281,10 @@ def load_run_graph( logging.error("Objects record not found: %s", objects_file) return None - with objects_file.open() as f: - all_definitions = [json.loads(line) for line in f if line.strip()] - all_configs_dict = ConfigInformation.load_objects( - all_definitions, as_instance=False - ) - - full_jobs = {} - for defn in all_definitions: - if "identifier" in defn: - jid = defn["identifier"] - config = all_configs_dict[defn["id"]] - full_jobs[jid] = config - if "init-tasks" in defn: - config.__xpm__.init_tasks = [ - all_configs_dict[tid] for tid in defn["init-tasks"] - ] - - dependencies = { - jid: get_all_dependencies(config, full_jobs) - for jid, config in full_jobs.items() - } - dependencies = {k: v for k, v in dependencies.items() if v} + graph = ParamsGraph.from_file(objects_file, load_docstrings=False) - job_tags_dict = {} + job_tags_dict: dict[str, dict] = {} + job_ids: set[str] = set() grid_keys: set[str] = set() jobs_jsonl = run_dir / "jobs.jsonl" if jobs_jsonl.exists(): @@ -212,11 +292,38 @@ def load_run_graph( for line in f: if line.strip(): job_data = json.loads(line) - if "job_id" in job_data and "tags" in job_data: - jtags = job_data["tags"] or {} - job_tags_dict[job_data["job_id"]] = jtags - if jtags: - grid_keys.update(jtags.keys()) + jid = job_data.get("job_id") + if jid: + job_ids.add(jid) + if "tags" in job_data: + jtags = job_data["tags"] or {} + job_tags_dict[jid] = jtags + if jtags: + grid_keys.update(jtags.keys()) + + status_data = load_run_status(run_dir) + action_ids: set[str] = set(status_data.get("actions", {}).keys()) + + all_configs_dict = _load_lightweight_objects( + graph, job_ids=job_ids, action_ids=action_ids + ) + + full_jobs = { + obj_node.identifier: all_configs_dict[obj_id] + for obj_id, obj_node in graph.objects.items() + if obj_node.identifier + } + + dependencies = { + jid: get_all_dependencies(config, full_jobs) + for jid, config in full_jobs.items() + } + # Remove PathGenerator back-edges where a Config's generated path points to + # the job directory of the enclosing Task that depends on that Config. + for jid, deps in list(dependencies.items()): + if not isinstance(full_jobs.get(jid), (Task, Action)): + dependencies[jid] = [d for d in deps if jid not in dependencies.get(d, ())] + dependencies = {k: v for k, v in dependencies.items() if v} # Populate tags for Config nodes (e.g. CNN architecture params) for jid, config in full_jobs.items(): From 003832d26434652a14e430f01e8baa183b9e455a Mon Sep 17 00:00:00 2001 From: Victor Morand Date: Tue, 20 Oct 2026 11:14:56 +0200 Subject: [PATCH 6/7] feat(dag): stack similar nodes based on configs and graph --- src/experimaestro/cli/dag.py | 8 + src/experimaestro/tests/test_xpm_dag.py | 356 +++++++++++++++ .../tools/inspect_params/__init__.py | 4 + .../tools/inspect_params/models.py | 8 +- src/experimaestro/tools/xpm_dag/__init__.py | 4 + .../tools/xpm_dag/dag_template.html | 431 +++++++++++++++++- src/experimaestro/tools/xpm_dag/draw_dag.py | 106 ++++- src/experimaestro/tools/xpm_dag/xpm_dag.py | 417 +++++++++++++++-- 8 files changed, 1265 insertions(+), 69 deletions(-) diff --git a/src/experimaestro/cli/dag.py b/src/experimaestro/cli/dag.py index a83bcf1c..2d2bf433 100644 --- a/src/experimaestro/cli/dag.py +++ b/src/experimaestro/cli/dag.py @@ -128,6 +128,12 @@ default="dagre", help="Layout algorithm for interactive web view: 'dagre' (default) or 'aligned'.", ) +@click.option( + "--stack/--no-stack", + "stack", + default=True, + help="Stack structurally equivalent nodes sharing class, parents, and children (default: enabled).", +) def dag( experiment_id: str | None, experiment_opt: str | None, @@ -149,6 +155,7 @@ def dag( filter_tags: str | None, include_actions: bool, layout: str, + stack: bool, ) -> None: """Visualize and export experiment dependency DAGs. @@ -185,6 +192,7 @@ def dag( filter_tags=filter_tags, include_actions=include_actions, layout=layout, + stack=stack, ) if exit_code != 0: raise click.exceptions.Exit(exit_code) diff --git a/src/experimaestro/tests/test_xpm_dag.py b/src/experimaestro/tests/test_xpm_dag.py index b59bf806..bd80dbfb 100644 --- a/src/experimaestro/tests/test_xpm_dag.py +++ b/src/experimaestro/tests/test_xpm_dag.py @@ -705,3 +705,359 @@ def test_load_run_graph_without_importing_remote_modules(tmp_path): dag_json = build_dag_json(full_jobs, dependencies, job_tags_dict, run_dir=run_dir) assert len(dag_json["nodes"]) == 5 + nodes_by_id = {n["id"]: n for n in dag_json["nodes"]} + assert nodes_by_id[out_cfg_id]["type"] == "init" + assert nodes_by_id[out_cfg_id]["status"] == "init" + assert nodes_by_id[out_cfg_id]["init"] is True + + +def test_init_task_node_type_and_filtering(tmp_path): + """Verify init_task nodes are typed as LightweightTask ('init') and kept by filter_tasks_only.""" + from experimaestro import LightweightTask + from experimaestro.tools.xpm_dag import ( + filter_tasks_only, + is_init_task_node, + load_run_graph, + ) + + run_dir = tmp_path / "init_run" + run_dir.mkdir(parents=True) + + init_id = "1" * 64 + cfg_id = "2" * 64 + task_id = "3" * 64 + + objects_data = [ + { + "id": 1, + "module": "pkg", + "type": "HFModelInitFromID", + "identifier": init_id, + "fields": {}, + }, + { + "id": 2, + "module": "pkg", + "type": "StaticConfig", + "identifier": cfg_id, + "fields": {}, + }, + { + "id": 3, + "module": "pkg", + "type": "Evaluate", + "identifier": task_id, + "task": 3, + "init-tasks": [1], + "fields": {"cfg": {"type": "python", "value": 2}}, + }, + ] + with (run_dir / "objects.jsonl").open("w") as f: + for obj in objects_data: + f.write(json.dumps(obj) + "\n") + with (run_dir / "jobs.jsonl").open("w") as f: + f.write( + json.dumps({"job_id": task_id, "task_id": "pkg.evaluate", "tags": {}}) + + "\n" + ) + + full_jobs, dependencies, job_tags_dict = load_run_graph(run_dir) + assert isinstance(full_jobs[init_id], LightweightTask) + assert is_init_task_node(full_jobs[init_id]) is True + assert is_init_task_node(full_jobs[cfg_id]) is False + assert is_init_task_node(full_jobs[task_id]) is False + + filtered_jobs, filtered_deps = filter_tasks_only(full_jobs, dependencies) + assert set(filtered_jobs.keys()) == {init_id, task_id} + assert init_id in filtered_deps[task_id] + + +def test_bottleneck_grid_config_detection(tmp_path): + """Verify get_experiment_configs detects core bottleneck grid configs when dataset fan-outs exist.""" + run_dir = tmp_path / "bottleneck_run" + run_dir.mkdir(parents=True) + + records = [ + # Upstream index builders (dataset/index fan-out) + { + "job_id": "idx1", + "task_id": "xpmir.index.sparse.sparseretrieverindexbuilder", + "tags": {"index_documents": "doc_a"}, + }, + { + "job_id": "idx2", + "task_id": "xpmir.index.sparse.sparseretrieverindexbuilder", + "tags": {"index_documents": "doc_b"}, + }, + # 1st-stage evaluations + { + "job_id": "eval1_a", + "task_id": "xpmir.evaluation.evaluate", + "tags": {"dataset": "ds_a", "first_stage": "splade"}, + }, + { + "job_id": "eval1_b", + "task_id": "xpmir.evaluation.evaluate", + "tags": {"dataset": "ds_b", "first_stage": "splade"}, + }, + # Core Learner grid-search jobs (2 configs) + { + "job_id": "learn1", + "task_id": "xpm_torch.learner.learner", + "tags": {"base": "ettin", "lr": 1e-5, "seed": 1}, + }, + { + "job_id": "learn2", + "task_id": "xpm_torch.learner.learner", + "tags": {"base": "ettin", "lr": 7e-6, "seed": 2}, + }, + # 2nd-stage evaluations (Cartesian product: 2 learners x 2 datasets = 4 jobs) + { + "job_id": "eval2_1a", + "task_id": "xpmir.evaluation.evaluate", + "tags": { + "base": "ettin", + "lr": 1e-5, + "seed": 1, + "dataset": "ds_a", + "first_stage": "splade", + "checkpoint": "last", + }, + }, + { + "job_id": "eval2_1b", + "task_id": "xpmir.evaluation.evaluate", + "tags": { + "base": "ettin", + "lr": 1e-5, + "seed": 1, + "dataset": "ds_b", + "first_stage": "splade", + "checkpoint": "last", + }, + }, + { + "job_id": "eval2_2a", + "task_id": "xpmir.evaluation.evaluate", + "tags": { + "base": "ettin", + "lr": 7e-6, + "seed": 2, + "dataset": "ds_a", + "first_stage": "splade", + "checkpoint": "last", + }, + }, + { + "job_id": "eval2_2b", + "task_id": "xpmir.evaluation.evaluate", + "tags": { + "base": "ettin", + "lr": 7e-6, + "seed": 2, + "dataset": "ds_b", + "first_stage": "splade", + "checkpoint": "last", + }, + }, + ] + + with (run_dir / "jobs.jsonl").open("w") as f: + for r in records: + f.write(json.dumps(r) + "\n") + + configs = get_experiment_configs(run_dir) + assert configs == [ + {"base": "ettin", "lr": 1e-5, "seed": 1}, + {"base": "ettin", "lr": 7e-6, "seed": 2}, + ] + + +def test_stack_dag_nodes_parallel_chains(): + """Verify stack_dag_nodes collapses parallel dataset chains while preserving Learner branches.""" + from experimaestro import LightweightTask + from experimaestro.tools.xpm_dag import stack_dag_nodes + + InitCls = type("HFModelInitFromID", (LightweightTask,), {}) + IdxCls = type("SparseRetrieverIndexBuilder", (Task,), {}) + EvalCls = type("Evaluate", (Task,), {}) + LearnCls = type("Learner", (Task,), {}) + LoaderCls = type("SimpleModuleLoader", (LightweightTask,), {}) + + full_jobs = { + "init_hf": InitCls(), + "idx_1": IdxCls(), + "idx_2": IdxCls(), + "idx_3": IdxCls(), + "eval1_1": EvalCls(), + "eval1_2": EvalCls(), + "eval1_3": EvalCls(), + "learn_a": LearnCls(), + "learn_b": LearnCls(), + "loader_a": LoaderCls(), + "loader_b": LoaderCls(), + "eval2_a1": EvalCls(), + "eval2_a2": EvalCls(), + "eval2_a3": EvalCls(), + "eval2_b1": EvalCls(), + "eval2_b2": EvalCls(), + "eval2_b3": EvalCls(), + } + dependencies = { + "idx_1": ["init_hf"], + "idx_2": ["init_hf"], + "idx_3": ["init_hf"], + "eval1_1": ["idx_1", "init_hf"], + "eval1_2": ["idx_2", "init_hf"], + "eval1_3": ["idx_3", "init_hf"], + "loader_a": ["learn_a"], + "loader_b": ["learn_b"], + "eval2_a1": ["loader_a", "eval1_1"], + "eval2_a2": ["loader_a", "eval1_2"], + "eval2_a3": ["loader_a", "eval1_3"], + "eval2_b1": ["loader_b", "eval1_1"], + "eval2_b2": ["loader_b", "eval1_2"], + "eval2_b3": ["loader_b", "eval1_3"], + } + job_tags_dict = { + "idx_1": {"index_documents": "d1"}, + "idx_2": {"index_documents": "d2"}, + "idx_3": {"index_documents": "d3"}, + "eval1_1": {"dataset": "d1", "first_stage": "splade"}, + "eval1_2": {"dataset": "d2", "first_stage": "splade"}, + "eval1_3": {"dataset": "d3", "first_stage": "splade"}, + "learn_a": {"lr": 1e-5, "seed": 1}, + "learn_b": {"lr": 7e-6, "seed": 2}, + "eval2_a1": {"lr": 1e-5, "seed": 1, "dataset": "d1", "first_stage": "splade"}, + "eval2_a2": {"lr": 1e-5, "seed": 1, "dataset": "d2", "first_stage": "splade"}, + "eval2_a3": {"lr": 1e-5, "seed": 1, "dataset": "d3", "first_stage": "splade"}, + "eval2_b1": {"lr": 7e-6, "seed": 2, "dataset": "d1", "first_stage": "splade"}, + "eval2_b2": {"lr": 7e-6, "seed": 2, "dataset": "d2", "first_stage": "splade"}, + "eval2_b3": {"lr": 7e-6, "seed": 2, "dataset": "d3", "first_stage": "splade"}, + } + + s_jobs, s_deps, s_tags, s_members = stack_dag_nodes( + full_jobs, dependencies, job_tags_dict + ) + # 17 nodes collapse into 9: + # init_hf (1), idx (x3), eval1 (x3), learn_a (1), learn_b (1), loader_a (1), loader_b (1), eval2_a (x3), eval2_b (x3) + assert len(s_jobs) == 9 + assert "learn_a" in s_jobs and "learn_b" in s_jobs + assert len(s_members["idx_1"]) == 3 + assert len(s_members["eval1_1"]) == 3 + assert len(s_members["eval2_a1"]) == 3 + assert len(s_members["eval2_b1"]) == 3 + assert s_tags["eval2_a1"] == { + "lr": 1e-5, + "seed": 1, + "dataset": "<3 values>", + "first_stage": "splade", + } + + dag_json = build_dag_json(full_jobs, dependencies, job_tags_dict) + html = generate_dag_html(dag_json) + assert 'id="btn-stack"' in html + assert "stack-backframe" in html + assert "stack-exterior-badge" in html + assert "type-init" in html + + +def test_bottleneck_configs_non_subset_intersection_and_upstream_filter(tmp_path): + """Verify cross-stage schema intersection config discovery (e.g. PlaidIndexBuilder -> Evaluate) + and downstream inclusion when filtering by an upstream-only tag.""" + from experimaestro.tools.xpm_dag.xpm_dag import ( + filter_subgraph_by_config, + get_experiment_configs, + stack_dag_nodes, + ) + + run_dir = tmp_path / "plaid_run" + run_dir.mkdir() + jobs_file = run_dir / "jobs.jsonl" + + models = ["lightonai/LateOn", "VictorMorand/Colbert-LateOn-Pearson"] + docs = ["doc_a", "doc_b", "doc_c"] + records = [] + full_jobs = {} + dependencies = {} + job_tags_dict = {} + + PlaidCls = type("PlaidIndexBuilder", (DummyTask,), {}) + EvalCls = type("Evaluate", (DummyTask,), {}) + + for m in models: + for d in docs: + idx_id = f"idx_{m}_{d}" + ev_id = f"eval_{m}_{d}" + idx_tags = { + "scorer": m, + "scorer_id": m, + "scorer_type": "stMultiVectorEncoder", + "index_documents": d, + } + ev_tags = { + "scorer": m, + "scorer_id": m, + "scorer_type": "stMultiVectorEncoder", + "dataset": f"ds_{d}", + "first_stage": "plaid", + } + records.append( + { + "job_id": idx_id, + "task_id": "xpmir.index.plaid.plaidindexbuilder", + "tags": idx_tags, + } + ) + records.append( + { + "job_id": ev_id, + "task_id": "xpmir.evaluation.evaluate", + "tags": ev_tags, + } + ) + full_jobs[idx_id] = PlaidCls() + full_jobs[ev_id] = EvalCls() + dependencies[ev_id] = [idx_id] + job_tags_dict[idx_id] = idx_tags + job_tags_dict[ev_id] = ev_tags + + with jobs_file.open("w") as f: + for r in records: + f.write(json.dumps(r) + "\n") + + # 1. Cross-stage intersection S_1 ∩ S_2 detects the 2 model configurations (not 12) + configs = get_experiment_configs(run_dir) + assert len(configs) == 2 + assert configs[0] == { + "scorer": "lightonai/LateOn", + "scorer_id": "lightonai/LateOn", + "scorer_type": "stMultiVectorEncoder", + } + assert configs[1] == { + "scorer": "VictorMorand/Colbert-LateOn-Pearson", + "scorer_id": "VictorMorand/Colbert-LateOn-Pearson", + "scorer_type": "stMultiVectorEncoder", + } + + # 2. Stacking with bottleneck grid_keys collapses 12 nodes into 4 (2 PlaidIndexBuilder x3 + 2 Evaluate x3) + grid_keys = set().union(*(c.keys() for c in configs)) + s_jobs, s_deps, s_tags, s_members = stack_dag_nodes( + full_jobs, dependencies, job_tags_dict, grid_keys=grid_keys + ) + assert len(s_jobs) == 4 + for rep_id, members in s_members.items(): + assert len(members) == 3 + + # 3. Filtering by an upstream-only tag (index_documents=doc_a) also includes downstream Evaluate + f_jobs, f_deps = filter_subgraph_by_config( + full_jobs, + dependencies, + job_tags_dict, + target_tags={"scorer": "lightonai/LateOn", "index_documents": "doc_a"}, + ) + assert set(f_jobs.keys()) == { + "idx_lightonai/LateOn_doc_a", + "eval_lightonai/LateOn_doc_a", + } + assert f_deps["eval_lightonai/LateOn_doc_a"] == ["idx_lightonai/LateOn_doc_a"] diff --git a/src/experimaestro/tools/inspect_params/__init__.py b/src/experimaestro/tools/inspect_params/__init__.py index 77168cdc..b8d3617f 100644 --- a/src/experimaestro/tools/inspect_params/__init__.py +++ b/src/experimaestro/tools/inspect_params/__init__.py @@ -26,6 +26,8 @@ ObjectNode, ParamsGraph, ResolvedPath, + normalize_identifier, + resolve_serialized_value, ) from experimaestro.utils.remote import ( clean_job_target, @@ -55,7 +57,9 @@ "find_remote_workspace_file", "find_workspace_file", "get_session_cache_dir", + "normalize_identifier", "resolve_job_file_pattern", + "resolve_serialized_value", "resolve_workspace", "run_inspector", "run_ssh_command", diff --git a/src/experimaestro/tools/inspect_params/models.py b/src/experimaestro/tools/inspect_params/models.py index 00c3ecaf..524ee4d2 100644 --- a/src/experimaestro/tools/inspect_params/models.py +++ b/src/experimaestro/tools/inspect_params/models.py @@ -149,6 +149,7 @@ class ObjectNode: outgoing_refs: List[Tuple[str, int]] = field(default_factory=list) docstring: Optional[str] = None task_id: Optional[int] = None + is_init_task: bool = False @classmethod def from_definition( @@ -249,6 +250,7 @@ def __init__( ) self.objects: Dict[int, ObjectNode] = {} self.task_object_ids: Set[int] = set() + self.init_task_object_ids: Set[int] = set() self.root_task_id: Optional[int] = None self._parse(raw_objs, load_docstrings=load_docstrings) @@ -306,7 +308,7 @@ def _parse( if node.task_id is not None: self.task_object_ids.add(node.task_id) for it in node.init_tasks: - self.task_object_ids.add(it) + self.init_task_object_ids.add(it) if target_hash and node.identifier == target_hash: cand_root = node.id @@ -314,6 +316,10 @@ def _parse( if tid in self.objects: self.objects[tid].is_task = True + for it_id in self.init_task_object_ids: + if it_id in self.objects and not self.objects[it_id].is_task: + self.objects[it_id].is_init_task = True + self.root_task_id = ( cand_root if cand_root else (int(raw_objs[-1]["id"]) if raw_objs else None) ) diff --git a/src/experimaestro/tools/xpm_dag/__init__.py b/src/experimaestro/tools/xpm_dag/__init__.py index 0cb09c8f..11a55635 100644 --- a/src/experimaestro/tools/xpm_dag/__init__.py +++ b/src/experimaestro/tools/xpm_dag/__init__.py @@ -19,12 +19,14 @@ filter_tasks_only, get_all_dependencies, get_experiment_configs, + is_init_task_node, load_dag_from_json, load_run_graph, load_run_status, parse_filter_arg, resolve_run_dir, resolve_target_config, + stack_dag_nodes, ) from .xpm_web_dag import ( generate_dag_html, @@ -45,6 +47,7 @@ "generate_dag_html", "get_all_dependencies", "get_experiment_configs", + "is_init_task_node", "load_dag_from_json", "load_run_graph", "load_run_status", @@ -56,4 +59,5 @@ "resolve_target_config", "run_dag", "serve_dag_web", + "stack_dag_nodes", ] diff --git a/src/experimaestro/tools/xpm_dag/dag_template.html b/src/experimaestro/tools/xpm_dag/dag_template.html index 702e6472..1cc1e8e2 100644 --- a/src/experimaestro/tools/xpm_dag/dag_template.html +++ b/src/experimaestro/tools/xpm_dag/dag_template.html @@ -307,6 +307,77 @@ border: 1px solid #cbd5e1; } + /* 1b. Init Tasks (LightweightTask) in Violet / Indigo */ + .node-card.type-init { + background: #f5f3ff; + border-color: #c4b5fd; + border-left: 4px solid #7c3aed; + } + .node-card.type-init .node-name { + color: #4c1d95; + } + .node-card.type-init .node-status-badge { + background: #ede9fe; + color: #6d28d9; + border: 1px solid #ddd6fe; + } + + /* Stacked node back-frames & exterior count label */ + .stack-backframe { + rx: 8px; + ry: 8px; + stroke-width: 1.5px; + transition: opacity 0.2s ease, stroke 0.15s ease; + } + .stack-backframe.type-task.status-done, .stack-backframe.type-action.status-done { + fill: #f0fdf4; + stroke: #86efac; + } + .stack-backframe.type-task.status-running, .stack-backframe.type-action.status-running { + fill: #fffbeb; + stroke: #fcd34d; + } + .stack-backframe.type-task.status-error, .stack-backframe.type-action.status-error { + fill: #fef2f2; + stroke: #fca5a5; + } + .stack-backframe.type-task.status-waiting, .stack-backframe.type-action.status-waiting { + fill: #f8fafc; + stroke: #cbd5e1; + } + .stack-backframe.type-task.status-ready, .stack-backframe.type-action.status-ready { + fill: #eff6ff; + stroke: #93c5fd; + } + .stack-backframe.type-init { + fill: #f5f3ff; + stroke: #c4b5fd; + } + .stack-backframe.type-config { + fill: #f8fafc; + stroke: #cbd5e1; + } + .node-group.dimmed .stack-backframe, + .node-group.dimmed .stack-exterior-badge { + opacity: 0.12; + } + .stack-exterior-pill { + fill: #1e293b; + stroke: #ffffff; + stroke-width: 1.5px; + rx: 6px; + ry: 6px; + } + .stack-exterior-text { + fill: #ffffff; + font-size: 11px; + font-weight: 700; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + dominant-baseline: central; + text-anchor: middle; + pointer-events: none; + } + /* 2. Tasks & Actions by Status */ .node-card.type-task.status-done, .node-card.type-action.status-done { background: #f0fdf4; @@ -562,6 +633,7 @@

    __TITLE__

    + @@ -617,7 +689,9 @@

    __TITLE__

    let currentOrientation = "LR"; // "LR" or "TB" let currentConfigIdx = (RAW_GRAPH.configs && RAW_GRAPH.configs.length > 0) ? "0" : "all"; let showConfigs = false; + let stackNodes = true; let selectedNodeId = null; + let currentRenderedNodes = []; // Node sizing constants matching flat plot aspect ratio const CARD_W = 240; @@ -770,6 +844,185 @@

    __TITLE__

    return reducedEdges; } + // Stack structurally equivalent nodes sharing (name, type, grid_tags, parent_groups, child_groups) + function stackGraphData(nodes, edges) { + if (nodes.length <= 1) return { nodes, edges }; + + const preds = new Map(); + const succs = new Map(); + nodes.forEach(n => { + preds.set(n.id, []); + succs.set(n.id, []); + }); + edges.forEach(e => { + if (preds.has(e.target)) preds.get(e.target).push(e.source); + if (succs.has(e.source)) succs.get(e.source).push(e.target); + }); + + // Determine bottleneck grid_keys to preserve core model branches in Full Grid view + const gridKeys = new Set(RAW_GRAPH.grid_keys || []); + + const baseSig = new Map(); + nodes.forEach(n => { + const tags = n.tags || {}; + const gPairs = []; + gridKeys.forEach(k => { + if (tags[k] !== undefined && tags[k] !== null) { + gPairs.push(`${k}=${tags[k]}`); + } + }); + gPairs.sort(); + baseSig.set(n.id, `${n.name}||${n.type}||${gPairs.join(",")}`); + }); + + let color = new Map(); + const initUniq = Array.from(new Set(baseSig.values())).sort(); + const initMap = new Map(initUniq.map((v, i) => [v, i])); + nodes.forEach(n => color.set(n.id, initMap.get(baseSig.get(n.id)))); + + for (let iter = 0; iter <= nodes.length; iter++) { + const nextSig = new Map(); + nodes.forEach(n => { + const pCols = Array.from(new Set((preds.get(n.id) || []).map(p => color.get(p)))).sort((a, b) => a - b); + const cCols = Array.from(new Set((succs.get(n.id) || []).map(c => color.get(c)))).sort((a, b) => a - b); + nextSig.set(n.id, `${baseSig.get(n.id)}||P:${pCols.join(",")}||C:${cCols.join(",")}`); + }); + const uniqSigs = Array.from(new Set(nextSig.values())).sort(); + const sigToId = new Map(uniqSigs.map((v, i) => [v, i])); + let changed = false; + const nextColor = new Map(); + nodes.forEach(n => { + const nc = sigToId.get(nextSig.get(n.id)); + if (nc !== color.get(n.id)) changed = true; + nextColor.set(n.id, nc); + }); + color = nextColor; + if (!changed) break; + } + + const groups = new Map(); + const sortedNodes = [...nodes].sort((a, b) => a.id.localeCompare(b.id)); + sortedNodes.forEach(n => { + const c = color.get(n.id); + if (!groups.has(c)) groups.set(c, []); + groups.get(c).push(n); + }); + + const memberToRep = new Map(); + const stackedNodes = []; + + groups.forEach(members => { + const rep = members[0]; + members.forEach(m => memberToRep.set(m.id, rep.id)); + + if (members.length === 1) { + stackedNodes.push({ + ...rep, + stack_count: 1, + stack_members: [rep], + }); + } else { + // Merge tags: keep shared values, summarize varying values as + const allTagKeys = []; + members.forEach(m => { + Object.keys(m.tags || {}).forEach(k => { + if (!allTagKeys.includes(k)) allTagKeys.push(k); + }); + }); + const mergedTags = {}; + const varyingTags = {}; + allTagKeys.forEach(k => { + const vals = members + .filter(m => m.tags && m.tags[k] !== undefined && m.tags[k] !== null) + .map(m => m.tags[k]); + const uniqVals = Array.from(new Set(vals.map(v => String(v)))); + if (uniqVals.length === 1 && vals.length === members.length) { + mergedTags[k] = vals[0]; + } else { + mergedTags[k] = `<${uniqVals.length} values>`; + varyingTags[k] = uniqVals; + } + }); + + // Merge depends_on (representative IDs) + const mergedDeps = []; + members.forEach(m => { + (m.depends_on || []).forEach(depId => { + const repDep = memberToRep.get(depId) || depId; + if (repDep !== rep.id && !mergedDeps.includes(repDep)) { + mergedDeps.push(repDep); + } + }); + }); + + stackedNodes.push({ + ...rep, + tags: mergedTags, + varying_tags: varyingTags, + depends_on: mergedDeps, + dependsOn: mergedDeps, + stack_count: members.length, + stack_members: members, + }); + } + }); + + // Normalize depends_on to mapped representative IDs + stackedNodes.forEach(sn => { + const mapped = []; + (sn.depends_on || []).forEach(d => { + const rd = memberToRep.get(d) || d; + if (rd !== sn.id && !mapped.includes(rd)) mapped.push(rd); + }); + sn.depends_on = mapped; + sn.dependsOn = mapped; + }); + + // Deduplicate edges between stacked representatives and transitively reduce + const seenEdges = new Set(); + const rawStackedEdges = []; + edges.forEach(e => { + const u = memberToRep.get(e.source) || e.source; + const v = memberToRep.get(e.target) || e.target; + if (u !== v) { + const k = `${u}->${v}`; + if (!seenEdges.has(k)) { + seenEdges.add(k); + rawStackedEdges.push({ source: u, target: v }); + } + } + }); + + // Remove transitive shortcuts on stacked edges + const sSuccs = new Map(); + stackedNodes.forEach(n => sSuccs.set(n.id, [])); + rawStackedEdges.forEach(e => { + if (sSuccs.has(e.source)) sSuccs.get(e.source).push(e.target); + }); + const reducedStackedEdges = []; + rawStackedEdges.forEach(e => { + let hasAlt = false; + const q = (sSuccs.get(e.source) || []).filter(c => c !== e.target); + const seen = new Set(q); + while (q.length > 0) { + const curr = q.pop(); + if (curr === e.target) { + hasAlt = true; + break; + } + for (const nxt of (sSuccs.get(curr) || [])) { + if (!seen.has(nxt)) { + seen.add(nxt); + q.push(nxt); + } + } + } + if (!hasAlt) reducedStackedEdges.push(e); + }); + + return { nodes: stackedNodes, edges: reducedStackedEdges }; + } + // Filter nodes and edges based on active filters function getFilteredData() { let nodes = [...RAW_GRAPH.nodes]; @@ -779,15 +1032,55 @@

    __TITLE__

    if (currentConfigIdx !== "all") { const targetConfig = RAW_GRAPH.configs[parseInt(currentConfigIdx, 10)]; if (targetConfig) { - // Find matching job nodes (tasks matching targetConfig, or configs whose tags match a subset of targetConfig) + const targetKeys = Object.keys(targetConfig); + // Match job nodes whose tags contain all targetConfig key-values (including downstream + // tasks with extra fan-out tags), or static config nodes matching a subset of targetConfig const matching = nodes.filter(n => { const tags = n.tags || {}; const tagKeys = Object.keys(tags); if (tagKeys.length === 0) return false; - return tagKeys.every(k => targetConfig[k] == tags[k]); + if (targetKeys.length > 0 && targetKeys.every(k => tags[k] !== undefined && tags[k] == targetConfig[k])) { + return true; + } + if (n.type === "config" && tagKeys.every(k => targetConfig[k] !== undefined && targetConfig[k] == tags[k])) { + return true; + } + return false; }); const selected = new Set(matching.map(n => n.id)); - const stack = [...matching.map(n => n.id)]; + const matchedTasks = matching.filter(n => n.type === "task"); + const matchedTaskIds = new Set(matchedTasks.map(n => n.id)); + const hasMultiStageMatch = matchedTasks.some(n => + (n.depends_on || []).some(d => matchedTaskIds.has(d)) + ); + + // Only traverse downstream tasks when matching hit only a single upstream stage + if (matchedTasks.length > 0 && !hasMultiStageMatch) { + const dependents = new Map(); + nodes.forEach(n => { + (n.depends_on || []).forEach(dep => { + if (!dependents.has(dep)) dependents.set(dep, []); + dependents.get(dep).push(n.id); + }); + }); + + const downStack = matchedTasks.map(n => n.id); + while (downStack.length > 0) { + const currId = downStack.pop(); + for (const childId of (dependents.get(currId) || [])) { + if (selected.has(childId)) continue; + const childNode = nodeMap.get(childId); + if (!childNode || childNode.type !== "task") continue; + const cTags = childNode.tags || {}; + if (targetKeys.every(k => cTags[k] === undefined || cTags[k] === null || cTags[k] == targetConfig[k])) { + selected.add(childId); + downStack.push(childId); + } + } + } + } + + const stack = [...selected]; // Traverse upstream dependencies while (stack.length > 0) { @@ -814,9 +1107,9 @@

    __TITLE__

    } } - // 2. Filter by Show Configs (default: hide static configs) + // 2. Filter by Show Configs (default: hide static configs, keep task, action, and init) if (!showConfigs) { - nodes = nodes.filter(n => n.type === "task" || n.type === "action"); + nodes = nodes.filter(n => n.type === "task" || n.type === "action" || n.type === "init" || n.init === true); } const activeNodeIds = new Set(nodes.map(n => n.id)); @@ -829,6 +1122,16 @@

    __TITLE__

    edges = computeTransitiveReduction(nodes, activeNodeIds); } + // 3. Apply structural node stacking if active + if (stackNodes) { + const stacked = stackGraphData(nodes, edges); + nodes = stacked.nodes; + edges = stacked.edges; + } else { + nodes = nodes.map(n => ({ ...n, stack_count: 1, stack_members: [n] })); + } + + currentRenderedNodes = nodes; return { nodes, edges }; } @@ -1217,7 +1520,23 @@

    __TITLE__

    nodeGroups.each(function(d) { const coords = nodeCoords.get(d.id) || { w: CARD_W, h: MIN_CARD_H }; - const fo = d3.select(this) + const groupEl = d3.select(this); + const scount = d.stack_count || 1; + + // Draw stacked offset back-frames behind the primary card + if (scount > 1) { + const nFrames = scount > 5 ? 3 : 2; + for (let layer = nFrames; layer >= 1; layer--) { + groupEl.append("rect") + .attr("class", `stack-backframe type-${d.type} status-${d.status}`) + .attr("x", layer * 5) + .attr("y", layer * 5) + .attr("width", coords.w) + .attr("height", coords.h); + } + } + + const fo = groupEl .append("foreignObject") .attr("width", coords.w) .attr("height", coords.h); @@ -1226,6 +1545,30 @@

    __TITLE__

    .attr("class", `node-card type-${d.type} status-${d.status}`) .attr("id", `card-${d.id}`); + // Exterior "x XX" badge outside the node stack + if (scount > 1) { + const nFrames = scount > 5 ? 3 : 2; + const badgeText = `× ${scount}`; + const pillW = Math.max(36, badgeText.length * 7 + 12); + const pillH = 20; + const extX = coords.w + nFrames * 5 + 6; + const extY = -4; + const extBadge = groupEl.append("g") + .attr("class", "stack-exterior-badge") + .attr("transform", `translate(${extX}, ${extY})`); + extBadge.append("rect") + .attr("class", "stack-exterior-pill") + .attr("x", 0) + .attr("y", 0) + .attr("width", pillW) + .attr("height", pillH); + extBadge.append("text") + .attr("class", "stack-exterior-text") + .attr("x", pillW / 2) + .attr("y", pillH / 2) + .text(badgeText); + } + // Header const header = card.append("div").attr("class", "node-header"); header.append("div") @@ -1234,14 +1577,14 @@

    __TITLE__

    .text(d.name); header.append("div") .attr("class", "node-status-badge") - .text(d.status); + .text(d.type === "init" ? "INIT" : d.status); - // Subline (Line 2: Job hash only) + // Subline (Line 2: Job hash + stacked indicator) const subline = card.append("div").attr("class", "node-subline"); const hashSpan = subline.append("span") .attr("class", "node-hash") - .attr("title", "Click to copy full ID") - .text(`[${d.id.substring(0, 8)}]`); + .attr("title", scount > 1 ? `${scount} stacked nodes (click to copy representative ID)` : "Click to copy full ID") + .text(scount > 1 ? `[${d.id.substring(0, 8)}] (+${scount - 1} stacked)` : `[${d.id.substring(0, 8)}]`); hashSpan.on("click", (e) => { e.stopPropagation(); copyText(d.id, "job ID"); @@ -1285,6 +1628,13 @@

    __TITLE__

    function highlightLineage(nodeId, edges) { const { upstream, downstream, upEdges, downEdges } = computeLineage(nodeId, edges); + d3.selectAll(".node-group").each(function(d) { + const isTarget = d.id === nodeId; + const isUp = upstream.has(d.id); + const isDown = downstream.has(d.id); + d3.select(this).classed("dimmed", !isTarget && !isUp && !isDown); + }); + d3.selectAll(".node-card").each(function(d) { const el = d3.select(this); const isTarget = d.id === nodeId; @@ -1313,6 +1663,7 @@

    __TITLE__

    } function clearHighlight() { + d3.selectAll(".node-group").classed("dimmed", false); d3.selectAll(".node-card") .classed("highlight-upstream", false) .classed("highlight-downstream", false) @@ -1340,40 +1691,68 @@

    __TITLE__

    // Side Inspector Drawer function openDrawer(nodeId) { - const node = RAW_GRAPH.nodes.find(n => n.id === nodeId); + const node = currentRenderedNodes.find(n => n.id === nodeId) || RAW_GRAPH.nodes.find(n => n.id === nodeId); if (!node) return; const drawer = document.getElementById("drawer"); const title = document.getElementById("drawer-title"); const body = document.getElementById("drawer-body"); + const scount = node.stack_count || 1; + const members = node.stack_members || [node]; - title.innerHTML = `${node.name} ${node.status}`; + title.innerHTML = `${node.name}${scount > 1 ? ` × ${scount}` : ""} ${node.type === "init" ? "INIT" : node.status}`; let tagsRows = ""; for (const [k, v] of Object.entries(node.tags || {})) { - tagsRows += `${k}${v}`; + const varList = node.varying_tags && node.varying_tags[k]; + const valDisplay = varList ? `${v}
    ${varList.slice(0, 8).join(", ")}${varList.length > 8 ? ", …" : ""}
    ` : `${v}`; + tagsRows += `${k}${valDisplay}`; } if (!tagsRows) tagsRows = `None`; // Parents / Dependencies let depsHtml = ""; (node.depends_on || []).forEach(depId => { - const pNode = RAW_GRAPH.nodes.find(n => n.id === depId); - const pName = pNode ? pNode.name : depId.substring(0, 8); + const pNode = currentRenderedNodes.find(n => n.id === depId) || RAW_GRAPH.nodes.find(n => n.id === depId); + const pName = pNode ? (pNode.stack_count > 1 ? `${pNode.name} × ${pNode.stack_count}` : pNode.name) : depId.substring(0, 8); depsHtml += `
    ${pName} ${depId.substring(0, 8)}
    `; }); if (!depsHtml) depsHtml = `
    No dependencies (Root)
    `; // Children / Dependents let succHtml = ""; - RAW_GRAPH.nodes.filter(n => (n.depends_on || []).includes(node.id)).forEach(child => { - succHtml += `
    ${child.name} ${child.id.substring(0, 8)}
    `; + currentRenderedNodes.filter(n => (n.depends_on || []).includes(node.id)).forEach(child => { + const cName = child.stack_count > 1 ? `${child.name} × ${child.stack_count}` : child.name; + succHtml += `
    ${cName} ${child.id.substring(0, 8)}
    `; }); if (!succHtml) succHtml = `
    No downstream dependents
    `; const jobPath = node.path || node.locator || ""; const displayPath = node.display_path || (jobPath ? '.../' + (node.task_id || node.name) + '/' + node.id.substring(0, 20) + '...' : ''); + let membersSectionHtml = ""; + if (scount > 1) { + const varyingKeys = Object.keys(node.varying_tags || {}); + const memberItems = members.map(m => { + const mTags = m.tags || {}; + const diffStr = varyingKeys.map(k => mTags[k] !== undefined ? `${k}=${mTags[k]}` : "").filter(Boolean).join(", "); + const mPath = m.path || m.locator || ""; + return `
    +
    + [${m.id.substring(0, 8)}] + 📋 Copy ${mPath ? "Path" : "ID"} +
    + ${diffStr ? `
    ${diffStr}
    ` : ""} +
    `; + }).join(""); + membersSectionHtml = ` +
    +
    Stacked Members (${scount})
    +
    ${memberItems}
    +
    + `; + } + body.innerHTML = `
    Overview
    @@ -1381,8 +1760,9 @@

    __TITLE__

    Job ID${node.id.substring(0, 16)}… 📋 Task ID${node.task_id || "—"} Type${node.type} - Status${node.status} - ${jobPath ? `Path${displayPath} 📋` : ''} + Status${node.type === "init" ? "init" : node.status} + ${scount > 1 ? `Stack Size× ${scount} nodes` : ''} + ${jobPath ? `Path${displayPath} 📋` : ''}
    @@ -1393,6 +1773,8 @@

    __TITLE__

    + ${membersSectionHtml} +
    Upstream Dependencies (${(node.depends_on || []).length})
    ${depsHtml}
    @@ -1478,6 +1860,17 @@

    __TITLE__

    renderGraph(); }); + const btnStack = document.getElementById("btn-stack"); + if (btnStack) { + btnStack.classList.toggle("active", stackNodes); + btnStack.addEventListener("click", () => { + stackNodes = !stackNodes; + btnStack.classList.toggle("active", stackNodes); + clearSelection(); + renderGraph(); + }); + } + const showConfigsToggle = document.getElementById("show-configs-toggle"); showConfigsToggle.checked = showConfigs; showConfigsToggle.addEventListener("change", (e) => { diff --git a/src/experimaestro/tools/xpm_dag/draw_dag.py b/src/experimaestro/tools/xpm_dag/draw_dag.py index 4c681232..cfad234d 100644 --- a/src/experimaestro/tools/xpm_dag/draw_dag.py +++ b/src/experimaestro/tools/xpm_dag/draw_dag.py @@ -56,12 +56,14 @@ filter_tasks_only, get_all_dependencies, get_experiment_configs, + is_init_task_node, load_dag_from_json, load_run_graph, load_run_status, parse_filter_arg, resolve_run_dir, resolve_target_config, + stack_dag_nodes, ) from .xpm_web_dag import ( generate_dag_html, @@ -82,6 +84,7 @@ "generate_dag_html", "get_all_dependencies", "get_experiment_configs", + "is_init_task_node", "load_dag_from_json", "load_run_graph", "load_run_status", @@ -92,6 +95,7 @@ "resolve_target_config", "run_dag", "serve_dag_web", + "stack_dag_nodes", ] @@ -106,6 +110,7 @@ def print_robust_tree( pos: dict[str, tuple[float, float]], prefix: str = "", is_last: bool = True, + stack_members: dict[str, list[str]] | None = None, ) -> None: """Recursively print the dependency tree with individual nodes, depth and Y info.""" if not job_ids: @@ -130,6 +135,10 @@ def job_sort_key(jid: str): ) depth = depths.get(jid, 0) y_coord = pos.get(jid, (0, 0))[1] + is_init = is_init_task_node(config) + members = (stack_members or {}).get(jid, [jid]) + stack_suffix = f" x {len(members)}" if len(members) > 1 else "" + init_badge = " [INIT]" if is_init else "" # Use tags from jobs.jsonl, fallback to tags(config) jtags = job_tags_dict.get(jid) @@ -140,7 +149,10 @@ def job_sort_key(jid: str): jtags = {} jtags = jtags or {} tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items()) - label = f"{job_type} [{jid[:8]}] (depth {depth}, y {y_coord:.1f})" + label = ( + f"{job_type}{init_badge}{stack_suffix} [{jid[:8]}] " + f"(depth {depth}, y {y_coord:.1f})" + ) if tag_str: label += f" ({tag_str})" @@ -169,6 +181,7 @@ def job_sort_key(jid: str): pos, child_prefix, True, + stack_members=stack_members, ) @@ -182,6 +195,7 @@ def plot_dag( title_suffix: str = "", show_plot: bool = True, save_path: Path | None = None, + stack_members: dict[str, list[str]] | None = None, ) -> None: """Plot the DAG hierarchically using pre-computed layout and reduced graph. @@ -195,6 +209,7 @@ def plot_dag( title_suffix: Optional suffix appended to figure title (e.g. config details). show_plot: Whether to display the plot interactively. save_path: Optional path to save the figure image. + stack_members: Optional mapping of representative job ID to stacked member IDs. """ try: import matplotlib.pyplot as plt @@ -217,6 +232,8 @@ def plot_dag( if isinstance(config, Action) else "task" if isinstance(config, Task) + else "init" + if is_init_task_node(config) else "config" ) @@ -229,12 +246,14 @@ def plot_dag( jtags = {} jtags = jtags or {} tag_str = ", ".join(f"{k}={v}" for k, v in jtags.items()) + members = (stack_members or {}).get(jid, [jid]) node_info[jid] = { - "name": name, + "name": f"{name} [INIT]" if ntype == "init" else name, "hash": f"[{jid[:8]}]", "tags": tag_str if tag_str else None, "type": ntype, + "stack_count": len(members), } # Calculate dynamic figure size based on the layout @@ -272,9 +291,45 @@ def plot_dag( ), ) - styles = {"config": "#e6ffed", "task": "#e6f4ff", "action": "#fff1f0"} + styles = { + "config": "#e6ffed", + "task": "#e6f4ff", + "init": "#f3e8ff", + "action": "#fff1f0", + } for jid, (x, y) in pos.items(): info = node_info[jid] + scount = info["stack_count"] + if scount > 1: + n_frames = 3 if scount > 5 else 2 + for layer in range(n_frames, 0, -1): + dx = layer * 0.35 + dy = -layer * 0.08 + ax.add_patch( + FancyBboxPatch( + (x - box_w / 2 + dx, y - box_h / 2 + dy), + box_w, + box_h, + boxstyle="round,pad=0.03,rounding_size=0.2", + facecolor=styles[info["type"]], + edgecolor="#888888", + linewidth=1.4, + alpha=0.85, + zorder=1.5, + ) + ) + plt.text( + x + box_w / 2 + (n_frames * 0.35) + 0.3, + y + box_h / 2 - 0.1, + f"x {scount}", + ha="left", + va="center", + fontsize=9.5, + fontweight="bold", + color="#334155", + zorder=4, + ) + ax.add_patch( FancyBboxPatch( (x - box_w / 2, y - box_h / 2), @@ -282,7 +337,7 @@ def plot_dag( box_h, boxstyle="round,pad=0.03,rounding_size=0.2", facecolor=styles[info["type"]], - edgecolor="#666666", + edgecolor="#7c3aed" if info["type"] == "init" else "#666666", linewidth=2.0, zorder=2, ) @@ -392,6 +447,7 @@ def run_dag( filter_tags: str | None = None, include_actions: bool = False, layout: str = "dagre", + stack: bool = True, ) -> int: """Execute the experiment DAG visualization/export pipeline with given options.""" save_plot_path = None @@ -511,23 +567,30 @@ def run_dag( # Optional terminal text tree visualization if tree: + t_jobs, t_deps, t_tags = full_jobs, dependencies, job_tags_dict + stack_members = None + if stack: + t_jobs, t_deps, t_tags, stack_members = stack_dag_nodes( + full_jobs, dependencies, job_tags_dict + ) dependents: dict[str, list[str]] = {} - for jid, deps in dependencies.items(): + for jid, deps in t_deps.items(): for d in deps: dependents.setdefault(d, []).append(jid) - depths, pos, _ = compute_dag_layout(full_jobs, dependencies, dependents) + depths, pos, _ = compute_dag_layout(t_jobs, t_deps, dependents) - tree_roots = [jid for jid in full_jobs if not dependencies.get(jid)] + tree_roots = [jid for jid in t_jobs if not t_deps.get(jid)] print_robust_tree( tree_roots, dependents, - dependencies, - full_jobs, - job_tags_dict, + t_deps, + t_jobs, + t_tags, set(), depths, pos, + stack_members=stack_members, ) if not (web or plot or save_plot_path): return 0 @@ -553,23 +616,30 @@ def run_dag( # Optional Matplotlib & graphical plot visualization (requires matplotlib) if plot or save_plot_path: + p_jobs, p_deps, p_tags = full_jobs, dependencies, job_tags_dict + stack_members = None + if stack: + p_jobs, p_deps, p_tags, stack_members = stack_dag_nodes( + full_jobs, dependencies, job_tags_dict + ) dependents = {} - for jid, deps in dependencies.items(): + for jid, deps in p_deps.items(): for d in deps: dependents.setdefault(d, []).append(jid) depths, pos, reduced_dag = compute_dag_layout( - full_jobs, dependencies, dependents + p_jobs, p_deps, dependents ) plot_dag( - full_jobs, - job_tags_dict, + p_jobs, + p_tags, pos, reduced_dag, title_suffix=title_suffix, show_plot=plot, save_path=save_plot_path, + stack_members=stack_members, ) return 0 @@ -692,6 +762,13 @@ def main() -> None: default="dagre", help="Layout algorithm for interactive web view: 'dagre' (default) or 'aligned' (stage-aligned flow).", ) + parser.add_argument( + "--no-stack", + action="store_false", + dest="stack", + default=True, + help="Disable stacking of structurally equivalent nodes.", + ) parser.add_argument( "--save-as", type=Path, @@ -722,6 +799,7 @@ def main() -> None: filter_tags=args.filter, include_actions=args.include_actions, layout=args.layout, + stack=args.stack, ) diff --git a/src/experimaestro/tools/xpm_dag/xpm_dag.py b/src/experimaestro/tools/xpm_dag/xpm_dag.py index d4f8c012..2740f11b 100644 --- a/src/experimaestro/tools/xpm_dag/xpm_dag.py +++ b/src/experimaestro/tools/xpm_dag/xpm_dag.py @@ -18,7 +18,7 @@ from typing import Any import networkx as nx -from experimaestro import Action, Config, Task, tags +from experimaestro import Action, Config, LightweightTask, Task, tags from experimaestro.scheduler.workspace_state_provider import WorkspaceStateProvider from experimaestro.settings import find_workspace, get_workspace from experimaestro.tools.inspect_params.models import ( @@ -188,14 +188,22 @@ def __init__(self, name: str): class _LightweightXPMInfo: """Lightweight metadata container matching ConfigInformation's DAG interface.""" - __slots__ = ("identifier", "task", "init_tasks", "dependencies", "values") + __slots__ = ( + "identifier", + "task", + "init_tasks", + "dependencies", + "values", + "is_init_task", + ) - def __init__(self, identifier: str): + def __init__(self, identifier: str, is_init_task: bool = False): self.identifier = identifier self.task: Any = None self.init_tasks: list[Any] = [] self.dependencies: list[Any] = [] self.values: dict[str, Any] = {} + self.is_init_task = is_init_task def xpmvalues(self): for k, v in self.values.items(): @@ -205,15 +213,27 @@ def tags(self) -> dict[str, Any]: return {} +def is_init_task_node(config: Any) -> bool: + """Return True if a node is a LightweightTask (init_task) and not a full Task or Action.""" + if config is None: + return False + if isinstance(config, (Task, Action)): + return False + if isinstance(config, LightweightTask): + return True + xpm = getattr(config, "__xpm__", None) + return bool(getattr(xpm, "is_init_task", False)) + + def _load_lightweight_objects( graph: ParamsGraph, job_ids: set[str], action_ids: set[str], ) -> dict[int, Any]: - """Construct lightweight Config, Task, and Action nodes from an indexed ParamsGraph. + """Construct lightweight Config, LightweightTask, Task, and Action nodes from an indexed ParamsGraph. Avoids importing external Python modules or executing remote file paths while - preserving class names, isinstance(node, (Task, Action, Config)) semantics, + preserving class names, isinstance(node, (Task, Action, LightweightTask, Config)) semantics, and __xpm__ dependency/parameter traversal. """ cls_cache: dict[tuple[type, str, str], type] = {} @@ -228,6 +248,8 @@ def _load_lightweight_objects( base_cls: type = Action elif (ident and ident in job_ids) or obj_node.is_task: base_cls = Task + elif obj_node.is_init_task: + base_cls = LightweightTask else: base_cls = Config # Fallback to already-imported module in sys.modules if present @@ -239,6 +261,8 @@ def _load_lightweight_objects( base_cls = Action elif issubclass(real_cls, Task): base_cls = Task + elif issubclass(real_cls, LightweightTask): + base_cls = LightweightTask cls_key = (base_cls, module_name, type_name) cls = cls_cache.get(cls_key) @@ -247,7 +271,11 @@ def _load_lightweight_objects( cls_cache[cls_key] = cls node = object.__new__(cls) - object.__setattr__(node, "__xpm__", _LightweightXPMInfo(ident)) + object.__setattr__( + node, + "__xpm__", + _LightweightXPMInfo(ident, is_init_task=(base_cls is LightweightTask)), + ) objects[obj_id] = node for obj_id, obj_node in graph.objects.items(): @@ -314,16 +342,51 @@ def load_run_graph( if obj_node.identifier } - dependencies = { - jid: get_all_dependencies(config, full_jobs) + direct_deps = { + jid: get_all_dependencies(config, full_jobs, recursive=False) for jid, config in full_jobs.items() } - # Remove PathGenerator back-edges where a Config's generated path points to - # the job directory of the enclosing Task that depends on that Config. - for jid, deps in list(dependencies.items()): + raw_all_deps = { + jid: set(get_all_dependencies(config, full_jobs, recursive=True)) + for jid, config in full_jobs.items() + } + # Remove PathGenerator back-edges on direct dependencies where a Config's + # generated path points to the job directory of an enclosing Task/Config. + for jid, deps in list(direct_deps.items()): if not isinstance(full_jobs.get(jid), (Task, Action)): - dependencies[jid] = [d for d in deps if jid not in dependencies.get(d, ())] - dependencies = {k: v for k, v in dependencies.items() if v} + direct_deps[jid] = [ + d for d in deps if jid not in raw_all_deps.get(d, set()) + ] + + # Expand transitive Config dependencies from cleaned direct_deps so nested + # Configs (e.g. ValidationSettings -> AggregatorValidationListener) never + # inherit stripped PathGenerator back-edges. + dependencies: dict[str, list[str]] = {} + for jid in full_jobs: + expanded: list[str] = [] + seen_deps: set[str] = set() + queue = list(direct_deps.get(jid, [])) + visited_cfg: set[str] = set() + while queue: + d = queue.pop(0) + if d == jid: + continue + if d not in seen_deps: + seen_deps.add(d) + expanded.append(d) + d_obj = full_jobs.get(d) + if ( + d_obj is not None + and not isinstance(d_obj, (Task, Action, LightweightTask)) + and not is_init_task_node(d_obj) + and d not in visited_cfg + ): + visited_cfg.add(d) + for sub_d in direct_deps.get(d, []): + if sub_d not in seen_deps: + queue.append(sub_d) + if expanded: + dependencies[jid] = expanded # Populate tags for Config nodes (e.g. CNN architecture params) for jid, config in full_jobs.items(): @@ -348,7 +411,9 @@ def load_run_status(run_dir: Path) -> dict[str, Any]: return {} -def get_all_dependencies(config, full_jobs: dict) -> list[str]: +def get_all_dependencies( + config, full_jobs: dict, *, recursive: bool = True +) -> list[str]: """Walk the config and its fields to find all dependencies.""" xpm = config.__xpm__ dep_ids = set() @@ -378,6 +443,8 @@ def walk_value(v): v_id = str(v.__xpm__.identifier) if v_id in full_jobs: dep_ids.add(v_id) + if not recursive: + return if v_id not in processed_configs: processed_configs.add(v_id) for _, sub_value in v.__xpm__.xpmvalues(): @@ -402,25 +469,109 @@ def walk_value(v): return list(dep_ids) +def _detect_bottleneck_configs( + job_records: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Detect core bottleneck grid-search configurations from jobs.jsonl records. + + When an experiment contains both core model grid-search tasks (e.g. Learner + tagged with ``{base, lr, seed}``) and dataset/evaluation fan-out tasks + (e.g. IndexBuilder tagged with ``{index_documents}``, 1st-stage Evaluate + tagged with ``{dataset, first_stage}``, and 2nd-stage Evaluate tagged with + ``{base, lr, seed, dataset, first_stage, checkpoint}``), this function + identifies the bottleneck grid-search schema so that configuration filtering + selects one complete model run (including all its evaluation datasets) rather + than individual dataset jobs. + """ + tagged_records = [r for r in job_records if r.get("tags")] + if not tagged_records: + return [] + + by_schema: dict[frozenset[str], list[dict[str, Any]]] = {} + schema_task_ids: dict[frozenset[str], set[str]] = {} + for r in tagged_records: + tdict = r["tags"] + schema = frozenset(tdict.keys()) + by_schema.setdefault(schema, []).append(tdict) + if r.get("task_id"): + schema_task_ids.setdefault(schema, set()).add(r["task_id"]) + + schemas = list(by_schema.keys()) + if len(schemas) > 1: + # Look for shared bottleneck schemas across distinct task stages via + # pairwise schema intersections I = S_a ∩ S_b (which covers both strict + # subsets S_a ⊂ S_b like Learner -> Evaluate and non-subset stage overlaps + # like IndexBuilder{model, doc} -> Evaluate{model, dataset}). + candidate_configs: dict[frozenset[str], list[dict[str, Any]]] = {} + for idx_a, s_a in enumerate(schemas): + tids_a = schema_task_ids.get(s_a, set()) + uniq_a = {tuple(sorted(d.items())) for d in by_schema[s_a]} + for s_b in schemas[idx_a + 1 :]: + tids_b = schema_task_ids.get(s_b, set()) + if tids_a and tids_b and tids_a == tids_b: + continue + inter = s_a & s_b + if not inter: + continue + uniq_b = {tuple(sorted(d.items())) for d in by_schema[s_b]} + proj_a = { + tuple(sorted((k, d[k]) for k in inter)) + for d in by_schema[s_a] + } + proj_b = { + tuple(sorted((k, d[k]) for k in inter)) + for d in by_schema[s_b] + } + shared_proj = proj_a & proj_b + union_proj = proj_a | proj_b + # Require shared bottleneck configurations across stages and a 1->N fan-out + if shared_proj and max(len(uniq_a), len(uniq_b)) > len(union_proj): + if inter not in candidate_configs: + configs_for_inter: list[dict[str, Any]] = [] + seen_inter: set[tuple] = set() + for d in by_schema[s_a] + by_schema[s_b]: + proj_dict = {k: d[k] for k in inter} + key = tuple(sorted(proj_dict.items())) + if key in shared_proj and key not in seen_inter: + seen_inter.add(key) + configs_for_inter.append(proj_dict) + if configs_for_inter: + candidate_configs[inter] = configs_for_inter + + if candidate_configs: + # Prefer largest bottleneck schema key-set, breaking ties by strongest compression (fewest configs) + target_schema = max( + candidate_configs.keys(), + key=lambda s: (len(s), -len(candidate_configs[s])), + ) + return candidate_configs[target_schema] + + # Default: deduplicate all non-empty tag dictionaries in order + configs = [] + seen = set() + for r in tagged_records: + tdict = r["tags"] + key = tuple(sorted(tdict.items())) + if key not in seen: + seen.add(key) + configs.append(tdict) + return configs + + def get_experiment_configs(run_dir: Path) -> list[dict]: - """Retrieve unique non-empty tag dictionaries from jobs.jsonl in a run directory.""" + """Retrieve unique bottleneck grid-search tag dictionaries from jobs.jsonl in a run directory.""" jobs_jsonl = run_dir / "jobs.jsonl" if not jobs_jsonl.exists(): return [] - configs: list[dict] = [] - seen: set[tuple] = set() + job_records: list[dict[str, Any]] = [] with jobs_jsonl.open() as f: for line in f: if not line.strip(): continue - d = json.loads(line) - tags_dict = d.get("tags") - if tags_dict: - key = tuple(sorted(tags_dict.items())) - if key not in seen: - seen.add(key) - configs.append(tags_dict) + job_records.append(json.loads(line)) + + configs = _detect_bottleneck_configs(job_records) if configs: param_keys = sorted(set().union(*(c.keys() for c in configs))) logging.info( @@ -468,9 +619,47 @@ def filter_subgraph_by_config( if not matching: return {}, {} - # 2. Traverse upstream ancestors (CNN, Datasets, ParameterLoader) + # 2. If filtering matched only upstream tasks (no matched task depends on another + # matched task, e.g. --filter index_documents=...), traverse downstream Task + # dependents whose tags do not conflict with target_tags. + from experimaestro import Action + selected_nodes = set(matching) - stack = list(matching) + matched_tasks = [ + jid for jid in matching if isinstance(full_jobs.get(jid), Task) + ] + matched_task_set = set(matched_tasks) + has_multi_stage_match = any( + any(d in matched_task_set for d in dependencies.get(jid, [])) + for jid in matched_tasks + ) + if matched_tasks and not has_multi_stage_match: + dependents: dict[str, list[str]] = {} + for jid, deps in dependencies.items(): + if jid not in full_jobs: + continue + for d in deps: + dependents.setdefault(d, []).append(jid) + + down_stack = list(matched_tasks) + while down_stack: + curr = down_stack.pop() + for child in dependents.get(curr, []): + if child in selected_nodes: + continue + cfg = full_jobs.get(child) + if not isinstance(cfg, Task): + continue + child_tags = job_tags_dict.get(child) or {} + if all( + child_tags.get(k) is None or child_tags.get(k) == v + for k, v in target_tags.items() + ): + selected_nodes.add(child) + down_stack.append(child) + + # 3. Traverse upstream ancestors (CNN, Datasets, ParameterLoader, FilterUnassessedTopics) + stack = list(selected_nodes) while stack: curr = stack.pop() for parent in dependencies.get(curr, []): @@ -478,16 +667,14 @@ def filter_subgraph_by_config( selected_nodes.add(parent) stack.append(parent) - # 3. If requested, include downstream actions and evaluated models that depend on matching jobs + # 4. If requested, include downstream actions and evaluated models that depend on matching jobs if include_actions: - from experimaestro import Action - for jid, config in full_jobs.items(): if isinstance(config, Action) or type(config).__name__.startswith( "EvaluatedModel" ): deps = dependencies.get(jid, []) - if any(m in deps for m in matching): + if any(m in deps for m in selected_nodes): selected_nodes.add(jid) # 4. Construct filtered jobs and dependencies restricted to selected_nodes @@ -503,10 +690,11 @@ def filter_subgraph_by_config( def filter_tasks_only( full_jobs: dict, dependencies: dict[str, list[str]] ) -> tuple[dict, dict[str, list[str]]]: - """Filter the graph to include only Task and Action nodes.""" - from experimaestro import Task, Action - - is_task = {jid: isinstance(c, (Task, Action)) for jid, c in full_jobs.items()} + """Filter the graph to include only Task, Action, and LightweightTask (init_task) nodes.""" + is_task = { + jid: isinstance(c, (Task, Action, LightweightTask)) or is_init_task_node(c) + for jid, c in full_jobs.items() + } task_deps: dict[str, list[str]] = {} for jid in full_jobs: if not is_task[jid]: @@ -526,6 +714,152 @@ def filter_tasks_only( return filtered_jobs, task_deps +def stack_dag_nodes( + full_jobs: dict, + dependencies: dict[str, list[str]], + job_tags_dict: dict[str, dict], + grid_keys: set[str] | None = None, +) -> tuple[dict, dict[str, list[str]], dict[str, dict], dict[str, list[str]]]: + """Stack structurally equivalent nodes sharing class name, grid tags, and parent/child groups. + + Uses iterative bisimulation refinement on the transitively reduced DAG so that + parallel chains (e.g. per-dataset index -> 1st-stage evaluate -> 2nd-stage evaluate) + collapse into stacked nodes while keeping distinct core grid-search branches separate. + + Returns: + Tuple of ``(stacked_jobs, stacked_dependencies, stacked_tags_dict, stack_members)``. + """ + if not full_jobs: + return {}, {}, {}, {} + + dag = nx.DiGraph() + for jid in full_jobs: + dag.add_node(jid) + for jid, deps in dependencies.items(): + if jid in dag: + for d in deps: + if d in dag and d != jid: + dag.add_edge(d, jid) + + reduced_dag = ( + nx.transitive_reduction(dag) + if nx.is_directed_acyclic_graph(dag) + else dag + ) + + # Infer bottleneck grid_keys if not explicitly supplied + if grid_keys is None: + mock_records = [ + { + "job_id": jid, + "task_id": type(c).__name__, + "tags": job_tags_dict.get(jid) or {}, + } + for jid, c in full_jobs.items() + if isinstance(c, Task) and job_tags_dict.get(jid) + ] + detected = _detect_bottleneck_configs(mock_records) + if detected: + # Only treat as distinguishing grid_keys if there are multiple schemas + all_keys = set().union(*(r["tags"].keys() for r in mock_records)) + det_keys = set().union(*(c.keys() for c in detected)) + if det_keys < all_keys: + grid_keys = det_keys + + base_sig: dict[str, tuple] = {} + for jid, config in full_jobs.items(): + cname = type(config).__name__.replace(".XPMConfig", "") + ntype = ( + "action" + if isinstance(config, Action) + else "task" + if isinstance(config, Task) + else "init" + if is_init_task_node(config) + else "config" + ) + jtags = job_tags_dict.get(jid) or {} + if grid_keys: + gtag_sig = tuple( + sorted((k, str(v)) for k, v in jtags.items() if k in grid_keys) + ) + else: + gtag_sig = () + base_sig[jid] = (cname, ntype, gtag_sig) + + # Iterative Weisfeiler-Lehman refinement on (base_sig, set(parent_colors), set(child_colors)) + uniq_init = {v: i for i, v in enumerate(sorted(set(base_sig.values()), key=str))} + color: dict[str, int] = {jid: uniq_init[base_sig[jid]] for jid in full_jobs} + + for _ in range(len(full_jobs) + 1): + new_sig: dict[str, tuple] = {} + for jid in full_jobs: + p_cols = tuple(sorted({color[p] for p in reduced_dag.predecessors(jid)})) + c_cols = tuple(sorted({color[c] for c in reduced_dag.successors(jid)})) + new_sig[jid] = (base_sig[jid], p_cols, c_cols) + uniq_map = {v: i for i, v in enumerate(sorted(set(new_sig.values()), key=str))} + new_color = {jid: uniq_map[new_sig[jid]] for jid in full_jobs} + if new_color == color: + break + color = new_color + + groups: dict[int, list[str]] = {} + for jid in sorted(full_jobs.keys()): + groups.setdefault(color[jid], []).append(jid) + + member_to_rep: dict[str, str] = {} + stacked_jobs: dict[str, Any] = {} + stack_members: dict[str, list[str]] = {} + stacked_tags_dict: dict[str, dict] = {} + + for _, members in groups.items(): + rep_id = members[0] + stacked_jobs[rep_id] = full_jobs[rep_id] + stack_members[rep_id] = members + for m in members: + member_to_rep[m] = rep_id + + if len(members) == 1: + if rep_id in job_tags_dict: + stacked_tags_dict[rep_id] = dict(job_tags_dict[rep_id]) + else: + all_tkeys: list[str] = [] + for m in members: + for k in (job_tags_dict.get(m) or {}).keys(): + if k not in all_tkeys: + all_tkeys.append(k) + merged_tags: dict[str, Any] = {} + for k in all_tkeys: + vals = [ + (job_tags_dict.get(m) or {}).get(k) + for m in members + if k in (job_tags_dict.get(m) or {}) + ] + uniq_vals = {str(v) for v in vals} + if len(uniq_vals) == 1 and len(vals) == len(members): + merged_tags[k] = vals[0] + else: + merged_tags[k] = f"<{len(uniq_vals)} values>" + if merged_tags: + stacked_tags_dict[rep_id] = merged_tags + + stacked_dependencies: dict[str, list[str]] = {} + for rep_id, members in stack_members.items(): + rep_deps: list[str] = [] + seen_deps: set[str] = {rep_id} + for m in members: + for d in dependencies.get(m, []): + if d in member_to_rep: + d_rep = member_to_rep[d] + if d_rep not in seen_deps: + seen_deps.add(d_rep) + rep_deps.append(d_rep) + if rep_deps: + stacked_dependencies[rep_id] = rep_deps + + return stacked_jobs, stacked_dependencies, stacked_tags_dict, stack_members + + def compute_dag_depths( dag: nx.DiGraph, full_jobs: dict, @@ -726,11 +1060,22 @@ def build_dag_json( name = type(config).__name__.replace(".XPMConfig", "") is_action = isinstance(config, Action) is_task = isinstance(config, Task) - ntype = "action" if is_action else "task" if is_task else "config" + is_init = is_init_task_node(config) + ntype = ( + "action" + if is_action + else "task" + if is_task + else "init" + if is_init + else "config" + ) # Determine task status if jid in job_states: status = job_states[jid] + elif is_init: + status = "init" elif is_task or is_action: status = "done" # If in finished run but not explicitly state-tracked else: @@ -772,6 +1117,7 @@ def build_dag_json( "taskId": task_id, "type": ntype, "status": status, + "init": is_init, "tags": jtags, # Dual dependency keys "depends_on": deps, @@ -800,6 +1146,7 @@ def build_dag_json( "nodes": nodes, "edges": edges, "configs": configs, + "grid_keys": sorted(grid_keys) if grid_keys else [], } From ebac83d9cc717aeeaf0b1f74ee7de3d1bc0f961c Mon Sep 17 00:00:00 2001 From: Victor Morand Date: Tue, 20 Oct 2026 14:11:16 +0200 Subject: [PATCH 7/7] fix: add networkx dependency, and fix dag complexity --- Makefile | 1 - docs/source/cli.md | 1 - docs/source/experiments/grid_search.md | 6 +- docs/source/img/icon-adaptive.svg | 2 +- pyproject.toml | 1 + src/experimaestro/cli/dag.py | 4 +- .../tests/test_grid_validation.py | 1 + src/experimaestro/tools/xpm_dag/draw_dag.py | 240 +++++++++++------- src/experimaestro/tools/xpm_dag/xpm_dag.py | 236 ++++++++++------- uv.lock | 161 +++++++----- 10 files changed, 402 insertions(+), 251 deletions(-) diff --git a/Makefile b/Makefile index 7005da0a..e9fa0e1a 100644 --- a/Makefile +++ b/Makefile @@ -21,4 +21,3 @@ $(ICON_LIGHT_BG): $(ICON_MASTER) scripts/make-app-icon.py $(ICON_ADAPTIVE): $(ICON_MASTER) scripts/make-app-icon.py uv run python scripts/make-app-icon.py --adaptive $(ICON_MASTER) $@ - diff --git a/docs/source/cli.md b/docs/source/cli.md index abc43d69..9672ea1e 100644 --- a/docs/source/cli.md +++ b/docs/source/cli.md @@ -582,4 +582,3 @@ experimaestro inspect -w my_cluster mymodule.MyTask/1822b2b | `e` | Expand all nodes | | `c` (on parent) | Collapse all child nodes | | `?` | Show interactive keyboard shortcuts and help | - diff --git a/docs/source/experiments/grid_search.md b/docs/source/experiments/grid_search.md index ae51afab..dac8eac7 100644 --- a/docs/source/experiments/grid_search.md +++ b/docs/source/experiments/grid_search.md @@ -140,15 +140,15 @@ class MyConfig(ConfigurationBase): def run(helper: ExperimentHelper, cfg: MyConfig): # Generate all permutations and their corresponding tags configurations, all_tags = generate_grid(cfg) - + for config, tags in zip(configurations, all_tags): # Each permutation is a finalized MyConfig object # 'tags' contains the specific values used (e.g., {'learning_rate': 0.1}) - + # Use tags to create a descriptive experiment ID tag_suffix = ".".join(f"{k}={v}" for k, v in tags.items()) sub_id = f"{config.id}.{tag_suffix}" if tag_suffix else config.id - + # Run your experiment logic for this config ... ``` diff --git a/docs/source/img/icon-adaptive.svg b/docs/source/img/icon-adaptive.svg index b5990c2d..51278331 100644 --- a/docs/source/img/icon-adaptive.svg +++ b/docs/source/img/icon-adaptive.svg @@ -7,4 +7,4 @@ [stroke='#026678' i] { stroke: #61b7c6 !important; } [stroke='#58c245' i] { stroke: #aaeb85 !important; } } - Experimaestro \ No newline at end of file + Experimaestro diff --git a/pyproject.toml b/pyproject.toml index 0307736c..62520ff7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ "typing-extensions >=4.2; python_version < \"3.12\"", "uvicorn[standard] >=0.27,<1", "watchdog >=2", + "networkx>=3.6.1", ] [project.optional-dependencies] diff --git a/src/experimaestro/cli/dag.py b/src/experimaestro/cli/dag.py index 2d2bf433..a2848350 100644 --- a/src/experimaestro/cli/dag.py +++ b/src/experimaestro/cli/dag.py @@ -5,8 +5,6 @@ from pathlib import Path import click -from experimaestro.tools.xpm_dag.draw_dag import run_dag - @click.command( name="dag", @@ -173,6 +171,8 @@ def dag( exp = experiment_id or experiment_opt json_out = str(json_file) if json_file else ("" if as_json else None) + from experimaestro.tools.xpm_dag.draw_dag import run_dag + exit_code = run_dag( experiment_id=exp, run_id=run_id, diff --git a/src/experimaestro/tests/test_grid_validation.py b/src/experimaestro/tests/test_grid_validation.py index bf42fd01..9ed5aebf 100644 --- a/src/experimaestro/tests/test_grid_validation.py +++ b/src/experimaestro/tests/test_grid_validation.py @@ -4,6 +4,7 @@ from experimaestro.utils.config import validate_attrs import pytest + @configuration() class SubConfig: value: GridSearch[int] = 1 diff --git a/src/experimaestro/tools/xpm_dag/draw_dag.py b/src/experimaestro/tools/xpm_dag/draw_dag.py index cfad234d..2d1cb68b 100644 --- a/src/experimaestro/tools/xpm_dag/draw_dag.py +++ b/src/experimaestro/tools/xpm_dag/draw_dag.py @@ -427,6 +427,140 @@ def plot_dag( plt.close() +def _resolve_save_as_destination( + save_as: Path, + html: Path | None, + json_out: str | None, + plot: bool, + web: bool | None, +) -> tuple[Path | None, str | None, Path | None, bool, bool | None] | None: + """Resolve file formats and destination paths for --save-as.""" + ext = save_as.suffix.lower() + save_plot_path = None + if ext in (".html", ".htm"): + html = save_as + web = False + elif ext == ".json": + json_out = str(save_as) + web = False + elif ext in (".png", ".pdf", ".svg", ".jpg", ".jpeg"): + save_plot_path = save_as + plot = False + web = False + else: + logging.error( + "Unsupported file extension '%s' for --save-as. " + "Supported extensions: .html, .htm, .json, .png, .pdf, .svg, .jpg", + ext, + ) + return None + return html, json_out, save_plot_path, plot, web + + +def _display_experiment_configs(run_dir: Path, exp_id: str, run_id: str) -> int: + """Display available configuration parameter combinations for an experiment run.""" + configs = get_experiment_configs(run_dir) + if not configs: + print(f"No configuration tags found for experiment '{exp_id}' ({run_id}).") # noqa: T201 + return 0 + print(f"Available configurations in experiment '{exp_id}' (run {run_id}):") # noqa: T201 + for idx, cfg_dict in enumerate(configs): + cfg_str = ", ".join(f"{k}={v}" for k, v in sorted(cfg_dict.items())) + print(f" [{idx}] {cfg_str}") # noqa: T201 + return 0 + + +def _export_dag_json(json_out: str, dag_json: dict) -> None: + """Export DAG structure as formatted JSON.""" + json_str = json.dumps(dag_json, indent=2) + if json_out: + out_path = Path(json_out) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json_str) + print(f"Saved DAG JSON to {out_path}") # noqa: T201 + else: + print(json_str) # noqa: T201 + + +def _export_dag_html(html: Path, dag_json: dict, exp_id: str, layout: str) -> None: + """Export standalone interactive HTML visualization page.""" + html_str = generate_dag_html( + dag_json, + title=f"Experimaestro DAG - {exp_id}", + default_layout=layout, + ) + html.parent.mkdir(parents=True, exist_ok=True) + html.write_text(html_str) + print(f"Saved interactive DAG HTML to {html}") # noqa: T201 + + +def _display_dag_tree( + full_jobs: dict, + dependencies: dict[str, list[str]], + job_tags_dict: dict[str, dict], + stack: bool, +) -> None: + """Render terminal text tree visualization.""" + t_jobs, t_deps, t_tags = full_jobs, dependencies, job_tags_dict + stack_members = None + if stack: + t_jobs, t_deps, t_tags, stack_members = stack_dag_nodes( + full_jobs, dependencies, job_tags_dict + ) + dependents: dict[str, list[str]] = {} + for jid, deps in t_deps.items(): + for d in deps: + dependents.setdefault(d, []).append(jid) + + depths, pos, _ = compute_dag_layout(t_jobs, t_deps, dependents) + tree_roots = [jid for jid in t_jobs if not t_deps.get(jid)] + print_robust_tree( + tree_roots, + dependents, + t_deps, + t_jobs, + t_tags, + set(), + depths, + pos, + stack_members=stack_members, + ) + + +def _display_or_save_dag_plot( + full_jobs: dict, + dependencies: dict[str, list[str]], + job_tags_dict: dict[str, dict], + title_suffix: str, + plot: bool, + save_plot_path: Path | None, + stack: bool, +) -> None: + """Render graphical matplotlib DAG plot or save it to image/vector file.""" + p_jobs, p_deps, p_tags = full_jobs, dependencies, job_tags_dict + stack_members = None + if stack: + p_jobs, p_deps, p_tags, stack_members = stack_dag_nodes( + full_jobs, dependencies, job_tags_dict + ) + dependents: dict[str, list[str]] = {} + for jid, deps in p_deps.items(): + for d in deps: + dependents.setdefault(d, []).append(jid) + + depths, pos, reduced_dag = compute_dag_layout(p_jobs, p_deps, dependents) + plot_dag( + p_jobs, + p_tags, + pos, + reduced_dag, + title_suffix=title_suffix, + show_plot=plot, + save_path=save_plot_path, + stack_members=stack_members, + ) + + def run_dag( *, experiment_id: str | None = None, @@ -452,24 +586,10 @@ def run_dag( """Execute the experiment DAG visualization/export pipeline with given options.""" save_plot_path = None if save_as: - ext = save_as.suffix.lower() - if ext in (".html", ".htm"): - html = save_as - web = False - elif ext == ".json": - json_out = str(save_as) - web = False - elif ext in (".png", ".pdf", ".svg", ".jpg", ".jpeg"): - save_plot_path = save_as - plot = False - web = False - else: - logging.error( - "Unsupported file extension '%s' for --save-as. " - "Supported extensions: .html, .htm, .json, .png, .pdf, .svg, .jpg", - ext, - ) + resolved = _resolve_save_as_destination(save_as, html, json_out, plot, web) + if resolved is None: return 1 + html, json_out, save_plot_path, plot, web = resolved # Resolve default web behavior: enabled by default unless --tree, --plot, or file export is active if web is None: @@ -484,15 +604,7 @@ def run_dag( exp_id = run_dir.parent.name if list_configs: - configs = get_experiment_configs(run_dir) - if not configs: - print(f"No configuration tags found for experiment '{exp_id}' ({run_id}).") # noqa: T201 - return 0 - print(f"Available configurations in experiment '{exp_id}' (run {run_id}):") # noqa: T201 - for idx, cfg_dict in enumerate(configs): - cfg_str = ", ".join(f"{k}={v}" for k, v in sorted(cfg_dict.items())) - print(f" [{idx}] {cfg_str}") # noqa: T201 - return 0 + return _display_experiment_configs(run_dir, exp_id, run_id) graph_data = load_run_graph(run_dir) if graph_data is None: @@ -534,14 +646,7 @@ def run_dag( run_id=run_id, workspace=workspace_path, ) - json_str = json.dumps(dag_json, indent=2) - if json_out: # Target file provided - out_path = Path(json_out) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json_str) - print(f"Saved DAG JSON to {out_path}") # noqa: T201 - else: - print(json_str) # noqa: T201 + _export_dag_json(json_out, dag_json) return 0 # Export standalone HTML page if requested @@ -555,43 +660,12 @@ def run_dag( run_id=run_id, workspace=workspace_path, ) - html_str = generate_dag_html( - dag_json, - title=f"Experimaestro DAG - {exp_id}", - default_layout=layout, - ) - html.parent.mkdir(parents=True, exist_ok=True) - html.write_text(html_str) - print(f"Saved interactive DAG HTML to {html}") # noqa: T201 + _export_dag_html(html, dag_json, exp_id, layout) return 0 # Optional terminal text tree visualization if tree: - t_jobs, t_deps, t_tags = full_jobs, dependencies, job_tags_dict - stack_members = None - if stack: - t_jobs, t_deps, t_tags, stack_members = stack_dag_nodes( - full_jobs, dependencies, job_tags_dict - ) - dependents: dict[str, list[str]] = {} - for jid, deps in t_deps.items(): - for d in deps: - dependents.setdefault(d, []).append(jid) - - depths, pos, _ = compute_dag_layout(t_jobs, t_deps, dependents) - - tree_roots = [jid for jid in t_jobs if not t_deps.get(jid)] - print_robust_tree( - tree_roots, - dependents, - t_deps, - t_jobs, - t_tags, - set(), - depths, - pos, - stack_members=stack_members, - ) + _display_dag_tree(full_jobs, dependencies, job_tags_dict, stack) if not (web or plot or save_plot_path): return 0 @@ -616,30 +690,14 @@ def run_dag( # Optional Matplotlib & graphical plot visualization (requires matplotlib) if plot or save_plot_path: - p_jobs, p_deps, p_tags = full_jobs, dependencies, job_tags_dict - stack_members = None - if stack: - p_jobs, p_deps, p_tags, stack_members = stack_dag_nodes( - full_jobs, dependencies, job_tags_dict - ) - dependents = {} - for jid, deps in p_deps.items(): - for d in deps: - dependents.setdefault(d, []).append(jid) - - depths, pos, reduced_dag = compute_dag_layout( - p_jobs, p_deps, dependents - ) - - plot_dag( - p_jobs, - p_tags, - pos, - reduced_dag, - title_suffix=title_suffix, - show_plot=plot, - save_path=save_plot_path, - stack_members=stack_members, + _display_or_save_dag_plot( + full_jobs, + dependencies, + job_tags_dict, + title_suffix, + plot, + save_plot_path, + stack, ) return 0 diff --git a/src/experimaestro/tools/xpm_dag/xpm_dag.py b/src/experimaestro/tools/xpm_dag/xpm_dag.py index 2740f11b..0f3360f0 100644 --- a/src/experimaestro/tools/xpm_dag/xpm_dag.py +++ b/src/experimaestro/tools/xpm_dag/xpm_dag.py @@ -515,12 +515,10 @@ def _detect_bottleneck_configs( continue uniq_b = {tuple(sorted(d.items())) for d in by_schema[s_b]} proj_a = { - tuple(sorted((k, d[k]) for k in inter)) - for d in by_schema[s_a] + tuple(sorted((k, d[k]) for k in inter)) for d in by_schema[s_a] } proj_b = { - tuple(sorted((k, d[k]) for k in inter)) - for d in by_schema[s_b] + tuple(sorted((k, d[k]) for k in inter)) for d in by_schema[s_b] } shared_proj = proj_a & proj_b union_proj = proj_a | proj_b @@ -625,9 +623,7 @@ def filter_subgraph_by_config( from experimaestro import Action selected_nodes = set(matching) - matched_tasks = [ - jid for jid in matching if isinstance(full_jobs.get(jid), Task) - ] + matched_tasks = [jid for jid in matching if isinstance(full_jobs.get(jid), Task)] matched_task_set = set(matched_tasks) has_multi_stage_match = any( any(d in matched_task_set for d in dependencies.get(jid, [])) @@ -714,24 +710,11 @@ def filter_tasks_only( return filtered_jobs, task_deps -def stack_dag_nodes( +def _build_reduced_dag( full_jobs: dict, dependencies: dict[str, list[str]], - job_tags_dict: dict[str, dict], - grid_keys: set[str] | None = None, -) -> tuple[dict, dict[str, list[str]], dict[str, dict], dict[str, list[str]]]: - """Stack structurally equivalent nodes sharing class name, grid tags, and parent/child groups. - - Uses iterative bisimulation refinement on the transitively reduced DAG so that - parallel chains (e.g. per-dataset index -> 1st-stage evaluate -> 2nd-stage evaluate) - collapse into stacked nodes while keeping distinct core grid-search branches separate. - - Returns: - Tuple of ``(stacked_jobs, stacked_dependencies, stacked_tags_dict, stack_members)``. - """ - if not full_jobs: - return {}, {}, {}, {} - +) -> nx.DiGraph: + """Build a directed graph from jobs and dependencies, applying transitive reduction if acyclic.""" dag = nx.DiGraph() for jid in full_jobs: dag.add_node(jid) @@ -740,54 +723,61 @@ def stack_dag_nodes( for d in deps: if d in dag and d != jid: dag.add_edge(d, jid) + return nx.transitive_reduction(dag) if nx.is_directed_acyclic_graph(dag) else dag - reduced_dag = ( - nx.transitive_reduction(dag) - if nx.is_directed_acyclic_graph(dag) - else dag + +def _infer_bottleneck_grid_keys( + full_jobs: dict, + job_tags_dict: dict[str, dict], +) -> set[str] | None: + """Infer distinguishing bottleneck grid keys if multiple schemas are present.""" + mock_records = [ + { + "job_id": jid, + "task_id": type(c).__name__, + "tags": job_tags_dict.get(jid) or {}, + } + for jid, c in full_jobs.items() + if isinstance(c, Task) and job_tags_dict.get(jid) + ] + detected = _detect_bottleneck_configs(mock_records) + if detected: + all_keys = set().union(*(r["tags"].keys() for r in mock_records)) + det_keys = set().union(*(c.keys() for c in detected)) + if det_keys < all_keys: + return det_keys + return None + + +def _compute_node_base_signature( + config: Any, + tags: dict, + grid_keys: set[str] | None, +) -> tuple: + """Compute base structural signature for a single job config.""" + cname = type(config).__name__.replace(".XPMConfig", "") + ntype = ( + "action" + if isinstance(config, Action) + else "task" + if isinstance(config, Task) + else "init" + if is_init_task_node(config) + else "config" ) + if grid_keys: + gtag_sig = tuple(sorted((k, str(v)) for k, v in tags.items() if k in grid_keys)) + else: + gtag_sig = () + return (cname, ntype, gtag_sig) - # Infer bottleneck grid_keys if not explicitly supplied - if grid_keys is None: - mock_records = [ - { - "job_id": jid, - "task_id": type(c).__name__, - "tags": job_tags_dict.get(jid) or {}, - } - for jid, c in full_jobs.items() - if isinstance(c, Task) and job_tags_dict.get(jid) - ] - detected = _detect_bottleneck_configs(mock_records) - if detected: - # Only treat as distinguishing grid_keys if there are multiple schemas - all_keys = set().union(*(r["tags"].keys() for r in mock_records)) - det_keys = set().union(*(c.keys() for c in detected)) - if det_keys < all_keys: - grid_keys = det_keys - - base_sig: dict[str, tuple] = {} - for jid, config in full_jobs.items(): - cname = type(config).__name__.replace(".XPMConfig", "") - ntype = ( - "action" - if isinstance(config, Action) - else "task" - if isinstance(config, Task) - else "init" - if is_init_task_node(config) - else "config" - ) - jtags = job_tags_dict.get(jid) or {} - if grid_keys: - gtag_sig = tuple( - sorted((k, str(v)) for k, v in jtags.items() if k in grid_keys) - ) - else: - gtag_sig = () - base_sig[jid] = (cname, ntype, gtag_sig) - # Iterative Weisfeiler-Lehman refinement on (base_sig, set(parent_colors), set(child_colors)) +def _refine_bisimulation_colors( + full_jobs: dict, + reduced_dag: nx.DiGraph, + base_sig: dict[str, tuple], +) -> dict[str, int]: + """Iterative Weisfeiler-Lehman refinement on (base_sig, set(parent_colors), set(child_colors)).""" uniq_init = {v: i for i, v in enumerate(sorted(set(base_sig.values()), key=str))} color: dict[str, int] = {jid: uniq_init[base_sig[jid]] for jid in full_jobs} @@ -802,17 +792,46 @@ def stack_dag_nodes( if new_color == color: break color = new_color + return color - groups: dict[int, list[str]] = {} - for jid in sorted(full_jobs.keys()): - groups.setdefault(color[jid], []).append(jid) +def _merge_member_tags( + members: list[str], + job_tags_dict: dict[str, dict], +) -> dict[str, Any]: + """Merge tags across stacked member nodes, noting multi-value variations.""" + all_tkeys: list[str] = [] + for m in members: + for k in (job_tags_dict.get(m) or {}).keys(): + if k not in all_tkeys: + all_tkeys.append(k) + merged_tags: dict[str, Any] = {} + for k in all_tkeys: + vals = [ + (job_tags_dict.get(m) or {}).get(k) + for m in members + if k in (job_tags_dict.get(m) or {}) + ] + uniq_vals = {str(v) for v in vals} + if len(uniq_vals) == 1 and len(vals) == len(members): + merged_tags[k] = vals[0] + else: + merged_tags[k] = f"<{len(uniq_vals)} values>" + return merged_tags + + +def _build_stacked_node_groups( + groups: dict[int, list[str]], + full_jobs: dict, + job_tags_dict: dict[str, dict], +) -> tuple[dict[str, Any], dict[str, list[str]], dict[str, dict], dict[str, str]]: + """Build representative nodes, member mappings, and merged tags for each color group.""" member_to_rep: dict[str, str] = {} stacked_jobs: dict[str, Any] = {} stack_members: dict[str, list[str]] = {} stacked_tags_dict: dict[str, dict] = {} - for _, members in groups.items(): + for members in groups.values(): rep_id = members[0] stacked_jobs[rep_id] = full_jobs[rep_id] stack_members[rep_id] = members @@ -823,26 +842,19 @@ def stack_dag_nodes( if rep_id in job_tags_dict: stacked_tags_dict[rep_id] = dict(job_tags_dict[rep_id]) else: - all_tkeys: list[str] = [] - for m in members: - for k in (job_tags_dict.get(m) or {}).keys(): - if k not in all_tkeys: - all_tkeys.append(k) - merged_tags: dict[str, Any] = {} - for k in all_tkeys: - vals = [ - (job_tags_dict.get(m) or {}).get(k) - for m in members - if k in (job_tags_dict.get(m) or {}) - ] - uniq_vals = {str(v) for v in vals} - if len(uniq_vals) == 1 and len(vals) == len(members): - merged_tags[k] = vals[0] - else: - merged_tags[k] = f"<{len(uniq_vals)} values>" - if merged_tags: - stacked_tags_dict[rep_id] = merged_tags + merged = _merge_member_tags(members, job_tags_dict) + if merged: + stacked_tags_dict[rep_id] = merged + return stacked_jobs, stack_members, stacked_tags_dict, member_to_rep + + +def _remap_stacked_dependencies( + stack_members: dict[str, list[str]], + dependencies: dict[str, list[str]], + member_to_rep: dict[str, str], +) -> dict[str, list[str]]: + """Remap dependencies using the representative node IDs and eliminate duplicates.""" stacked_dependencies: dict[str, list[str]] = {} for rep_id, members in stack_members.items(): rep_deps: list[str] = [] @@ -856,6 +868,50 @@ def stack_dag_nodes( rep_deps.append(d_rep) if rep_deps: stacked_dependencies[rep_id] = rep_deps + return stacked_dependencies + + +def stack_dag_nodes( + full_jobs: dict, + dependencies: dict[str, list[str]], + job_tags_dict: dict[str, dict], + grid_keys: set[str] | None = None, +) -> tuple[dict, dict[str, list[str]], dict[str, dict], dict[str, list[str]]]: + """Stack structurally equivalent nodes sharing class name, grid tags, and parent/child groups. + + Uses iterative bisimulation refinement on the transitively reduced DAG so that + parallel chains (e.g. per-dataset index -> 1st-stage evaluate -> 2nd-stage evaluate) + collapse into stacked nodes while keeping distinct core grid-search branches separate. + + Returns: + Tuple of ``(stacked_jobs, stacked_dependencies, stacked_tags_dict, stack_members)``. + """ + if not full_jobs: + return {}, {}, {}, {} + + reduced_dag = _build_reduced_dag(full_jobs, dependencies) + + if grid_keys is None: + grid_keys = _infer_bottleneck_grid_keys(full_jobs, job_tags_dict) + + base_sig = { + jid: _compute_node_base_signature( + config, job_tags_dict.get(jid) or {}, grid_keys + ) + for jid, config in full_jobs.items() + } + color = _refine_bisimulation_colors(full_jobs, reduced_dag, base_sig) + + groups: dict[int, list[str]] = {} + for jid in sorted(full_jobs.keys()): + groups.setdefault(color[jid], []).append(jid) + + stacked_jobs, stack_members, stacked_tags_dict, member_to_rep = ( + _build_stacked_node_groups(groups, full_jobs, job_tags_dict) + ) + stacked_dependencies = _remap_stacked_dependencies( + stack_members, dependencies, member_to_rep + ) return stacked_jobs, stacked_dependencies, stacked_tags_dict, stack_members diff --git a/uv.lock b/uv.lock index f67c0513..5811ab6f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -720,6 +720,8 @@ dependencies = [ { name = "humanfriendly" }, { name = "jinja2" }, { name = "marshmallow" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "networkx", version = "3.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "psutil" }, { name = "pydantic" }, { name = "pyparsing" }, @@ -798,6 +800,7 @@ requires-dist = [ { name = "marshmallow", specifier = ">=3.20,<4" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0" }, { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=2.0" }, + { name = "networkx", specifier = ">=3.6.1" }, { name = "paramiko", marker = "extra == 'ssh'", specifier = ">=3.3" }, { name = "psutil", specifier = ">=7,<8" }, { name = "pydantic", specifier = ">=2.6" }, @@ -1867,6 +1870,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/ca/36339329c4604adbcc99c899b7eb1ce1a555c499b6a6860757dc9bfed36d/narwhals-2.22.1-py3-none-any.whl", hash = "sha256:60567d774edf77db53906f89d9fbd164e66e56d66d388e1e6990f17ac33cfb53", size = 454815, upload-time = "2026-06-05T12:34:32.289Z" }, ] +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "networkx" +version = "3.7" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz", hash = "sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", size = 1866482, upload-time = "2026-09-21T16:45:16.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl", hash = "sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", size = 2142205, upload-time = "2026-09-21T16:45:14.609Z" }, +] + [[package]] name = "numpy" version = "2.4.6" @@ -2480,7 +2517,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } wheels = [ @@ -2503,7 +2540,7 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/fdedc2c75c3e31a330659c85e5793bb18b3397981fbf0844c6dee5b18926/pyee-14.0.0.tar.gz", hash = "sha256:76dd0f4314ecd27f02dc73589dea7fd3853f9b6176d8ef9b122860657e3602de", size = 98760, upload-time = "2026-08-13T04:26:11.021Z" } wheels = [ @@ -2887,13 +2924,13 @@ name = "scikit-learn" version = "1.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl" }, + { name = "joblib", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "narwhals", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "threadpoolctl", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -2917,7 +2954,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -2957,7 +2994,7 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -3043,23 +3080,23 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "roman-numerals" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, + { name = "alabaster", marker = "python_full_version < '3.12'" }, + { name = "babel", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, + { name = "docutils", marker = "python_full_version < '3.12'" }, + { name = "imagesize", marker = "python_full_version < '3.12'" }, + { name = "jinja2", marker = "python_full_version < '3.12'" }, + { name = "packaging", marker = "python_full_version < '3.12'" }, + { name = "pygments", marker = "python_full_version < '3.12'" }, + { name = "requests", marker = "python_full_version < '3.12'" }, + { name = "roman-numerals", marker = "python_full_version < '3.12'" }, + { name = "snowballstemmer", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.12'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } wheels = [ @@ -3082,23 +3119,23 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "roman-numerals" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, + { name = "alabaster", marker = "python_full_version >= '3.12'" }, + { name = "babel", marker = "python_full_version >= '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, + { name = "docutils", marker = "python_full_version >= '3.12'" }, + { name = "imagesize", marker = "python_full_version >= '3.12'" }, + { name = "jinja2", marker = "python_full_version >= '3.12'" }, + { name = "packaging", marker = "python_full_version >= '3.12'" }, + { name = "pygments", marker = "python_full_version >= '3.12'" }, + { name = "requests", marker = "python_full_version >= '3.12'" }, + { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, + { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ @@ -3324,7 +3361,7 @@ name = "typeguard" version = "4.5.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/1c/dfba5c4633cafc4c701f237d2ba63b416805047fd6d96aab4cfc40969f98/typeguard-4.5.2.tar.gz", hash = "sha256:5a16dcac23502039299c97c8941651bc33d7ea8cc4b2f7d6bbb1b528f6eea423", size = 80240, upload-time = "2026-05-14T12:59:40.857Z" } wheels = [ @@ -3372,9 +3409,9 @@ name = "tyro" version = "1.0.15" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "docstring-parser" }, - { name = "typeguard" }, - { name = "typing-extensions" }, + { name = "docstring-parser", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "typeguard", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "typing-extensions", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/12/78/a5749a6c1ee9abc2999e294f339f8f72476d1a60bb95fc0e86156aafed3b/tyro-1.0.15.tar.gz", hash = "sha256:3f1d60887723eecb9c489f195d11f079c4a1f33df74b723552ad31ec57c667bb", size = 593822, upload-time = "2026-06-20T08:48:28.364Z" } wheels = [ @@ -3857,17 +3894,17 @@ name = "zeus" version = "0.15.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "amdsmi" }, - { name = "httpx" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "nvidia-ml-py" }, - { name = "pandas" }, - { name = "pydantic" }, - { name = "python-dateutil" }, - { name = "rich" }, - { name = "scikit-learn" }, - { name = "tyro" }, + { name = "amdsmi", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "httpx", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-ml-py", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "pandas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "pydantic", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "python-dateutil", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "rich", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "scikit-learn", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "tyro", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/a1/29f935d77a45cf5174db3c42f9f4ab439bb03d13f69a7748ec95d3a94509/zeus-0.15.1.tar.gz", hash = "sha256:f7b347a93dc729c9dfa2379bbfe25c45fe58e659223a0930e76701b1441c5ca9", size = 227120, upload-time = "2026-03-29T05:44:14.433Z" } wheels = [ @@ -3876,7 +3913,7 @@ wheels = [ [package.optional-dependencies] apple = [ - { name = "zeus-apple-silicon" }, + { name = "zeus-apple-silicon", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] [[package]]