Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ jobs:
broad=true
performance=true
;;
src/sqlbuild/cli/commands/_helpers/lineage/*|src/sqlbuild/cli/commands/_helpers/plan/*|src/sqlbuild/cli/commands/_helpers/scope/*|src/sqlbuild/cli/commands/main/inspection/*|src/sqlbuild/cli/commands/main/project/*|src/sqlbuild/cli/output/*|src/sqlbuild/presentation/*)
# Inspection command guards run inside the fresh-process compile jobs.
broad=true
performance=true
;;
src/sqlbuild/rule_engine/*|src/sqlbuild/lint/*|scripts/rules_benchmark/*|scripts/benchmark_rules_ci.py|tests/performance/*)
broad=true
performance=true
Expand Down
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ test-e2e-cold-compile-performance:
test -n "$(SQLBUILD_BENCHMARK_MODELS)"
env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \
tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py \
tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py \
-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)" \
-vv --log-cli-level=INFO --color=yes

Expand Down
15 changes: 15 additions & 0 deletions tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,21 @@ class DagJsonBuildE2ETestCase:
expected_absent_fragments: tuple[str, ...]


@dataclass(frozen=True)
class DiamondDagE2ETestCase:
description: str
command: tuple[str, ...]
expected_fragments: tuple[str, ...]
expected_max_lines_per_node: int


@dataclass(frozen=True)
class DiamondDagJsonE2ETestCase:
description: str
expected_node_ids: tuple[str, ...]
expected_edge_count: int


@dataclass(frozen=True)
class PlanCommandBuildE2ETestCase:
"""Test case for plan command surface behavior."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@

from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import (
DagJsonBuildE2ETestCase,
DiamondDagE2ETestCase,
DiamondDagJsonE2ETestCase,
)
from tests.e2e.src.sqlbuild.cli.commands.main.lineage.helpers import (
DIAMOND_EDGE_COUNT,
diamond_model_names,
prepare_diamond_lineage_project,
)
from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_waffle_shop, run_sqb

Expand Down Expand Up @@ -84,3 +91,73 @@ def test_given_waffle_shop_when_running_dag_json_then_it_reports_static_graph(
assert fragment not in result.stdout
assert "query_sql" not in result.stdout
assert "action" not in result.stdout


@pytest.mark.parametrize(
"test_case",
[
DiamondDagE2ETestCase(
description="dag text summarizes a layered diamond graph",
command=("--no-color", "dag"),
expected_fragments=(
f"DAG ready ({len(diamond_model_names())} nodes, "
f"{DIAMOND_EDGE_COUNT} edges, 0 checks)",
),
expected_max_lines_per_node=1,
),
DiamondDagE2ETestCase(
description="dag json lists a layered diamond graph once per node and edge",
command=("--no-color", "dag", "--json"),
expected_fragments=('"project_name": "diamond_lineage"',),
expected_max_lines_per_node=40,
),
],
ids=lambda case: case.description,
)
def test_given_layered_diamonds_when_running_dag_then_output_stays_linear(
test_case: DiamondDagE2ETestCase,
tmp_path: Path,
) -> None:
project_dir: Path = prepare_diamond_lineage_project(tmp_path=tmp_path)

result: subprocess.CompletedProcess[str] = run_sqb(
command=test_case.command, project_dir=project_dir
)

assert result.returncode == 0, result.stdout + result.stderr
for fragment in test_case.expected_fragments:
assert fragment in result.stdout, result.stdout
assert len(result.stdout.splitlines()) <= test_case.expected_max_lines_per_node * len(
diamond_model_names()
)


@pytest.mark.parametrize(
"test_case",
[
DiamondDagJsonE2ETestCase(
description="dag json lists every diamond model and dependency exactly once",
expected_node_ids=tuple(f"model:{name}" for name in diamond_model_names()),
expected_edge_count=DIAMOND_EDGE_COUNT,
)
],
ids=lambda case: case.description,
)
def test_given_layered_diamonds_when_running_dag_json_then_lists_each_node_and_edge_once(
test_case: DiamondDagJsonE2ETestCase,
tmp_path: Path,
) -> None:
project_dir: Path = prepare_diamond_lineage_project(tmp_path=tmp_path)

result: subprocess.CompletedProcess[str] = run_sqb(
command=("--no-color", "dag", "--json"), project_dir=project_dir
)

assert result.returncode == 0, result.stdout + result.stderr
payload: dict[str, list[dict[str, object]]] = json.loads(result.stdout)
node_ids: list[str] = [str(node["id"]) for node in payload["nodes"]]
edge_pairs: list[tuple[str, str]] = [
(str(edge["from_id"]), str(edge["to_id"])) for edge in payload["edges"]
]
assert sorted(node_ids) == sorted(test_case.expected_node_ids)
assert len(edge_pairs) == len(set(edge_pairs)) == test_case.expected_edge_count
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,19 @@ Both guards initialize the process-local SQL runtime with an unrelated 32-model
measurement. “Cold” means the measured project has no compiler cache or target artifacts; it does
not include one-time lazy initialization noise from the Python test worker. Phase timings are
attribution spans rather than additive buckets.

## Inspection command guard

The semantic benchmark's model graph is a spine plus independent chains, so it has no shared
dependencies. `test_inspection_command_performance.py` adds an opt-in lattice to the 3,000-model
semantic project: `shared_orders_hub` reads the end of the spine and fans out to eight models,
sixteen layers each join two neighbouring slots of the previous layer, and `shared_orders_rollup`
fans the last layer back in. Every lattice model is reachable along exponentially many paths. The
compile guards do not enable the lattice, so their fingerprints and budgets are unchanged.

After one compile and warm lineage and scope caches, the guard runs `sqb lineage` downstream from
the hub and upstream from the rollup as text and JSON, `sqb dag --json` and `sqb scope --json`,
each in a fresh process. Every command
has a wall-time ceiling, a peak-RSS ceiling and an output-line bound linear in the model count, so
a renderer or graph walk that re-expands shared nodes per path fails on size before it can hide
behind a loose timing budget. It runs in the 3,000-model fresh-process compile job.
10 changes: 10 additions & 0 deletions tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,3 +279,13 @@ class DiagnosticPerformanceCase:
expected_max_wall_seconds: float = 20.0
expected_timeout_seconds: float = 65.0
diagnostic_count: int = 100


@dataclass(frozen=True)
class InspectionCommandPerformanceGuardTestCase:
description: str
sqb_args: tuple[str, ...]
expected_fragments: tuple[str, ...]
expected_max_output_lines: int
expected_max_wall_seconds: float
expected_max_rss_bytes: int
14 changes: 14 additions & 0 deletions tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@

import pytest

from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import (
assert_required_cgroup_memory_limit,
)
from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import (
prepare_inspection_benchmark_project,
)
from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb


Expand All @@ -21,3 +27,11 @@ def semantic_playground(tmp_path_factory: pytest.TempPathFactory) -> Path:
typed.read_text(encoding="utf-8"), encoding="utf-8"
)
return project


@pytest.fixture(scope="module")
def inspection_benchmark_project(tmp_path_factory: pytest.TempPathFactory) -> Path:
assert_required_cgroup_memory_limit()
project_dir: Path = tmp_path_factory.mktemp("inspection") / "semantic_inspection"
prepare_inspection_benchmark_project(project_dir=project_dir)
return project_dir
Loading
Loading