Skip to content

Commit a8dff6d

Browse files
committed
test(compiler): prove compile-input stage captures are complete
Plan a generated project with every feature block and require its CompileProjectInputs capture to hold every field canonically, name every callable, and prove every required render kind. Perturb the native compile inputs to show the differential localises a render mismatch to the compile_project_inputs capture.
1 parent f12e642 commit a8dff6d

4 files changed

Lines changed: 137 additions & 0 deletions

File tree

‎tests/e2e/scripts/compiler_differential/_test_types.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,14 @@ class CoverageFailureTestCase:
5959
expected_exit_code: int
6060
expected_lines: tuple[str, ...]
6161
expected_absent: tuple[str, ...]
62+
63+
64+
@dataclass(frozen=True)
65+
class RenderCaptureTestCase:
66+
"""A generated project with every feature block and the render kinds its capture proves."""
67+
68+
description: str
69+
seed: int
70+
blocks: tuple[str, ...]
71+
command: tuple[str, ...]
72+
expected_kinds: frozenset[str]

‎tests/e2e/scripts/compiler_differential/helpers.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,27 @@ def _perturbed(*, until, **stages):
4949
return dataclasses.replace(result, model_files=tuple(reversed(result.model_files)))
5050
5151
52+
frontier.native_frontier = _perturbed
53+
"""
54+
55+
RENDER_PERTURBATION: str = """
56+
import dataclasses
57+
58+
import sqlbuild.compiler.frontier.main._compile_frontier as frontier
59+
from sqlbuild.compiler.frontier.types import CompilerStage
60+
61+
_original = frontier.native_frontier
62+
63+
64+
def _perturbed(*, until, **stages):
65+
result = _original(until=until, **stages)
66+
if until is not CompilerStage.COMPILE_PROJECT_INPUTS or not result.model_inputs:
67+
return result
68+
first = result.model_inputs[0]
69+
changed = dataclasses.replace(first, macro_deps=(*first.macro_deps, "perturbed_by_test"))
70+
return dataclasses.replace(result, model_inputs=(changed, *result.model_inputs[1:]))
71+
72+
5273
frontier.native_frontier = _perturbed
5374
"""
5475

‎tests/e2e/scripts/compiler_differential/test_compiler_differential.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from tests.e2e.scripts.compiler_differential.helpers import (
1717
DISCOVERY_PERTURBATION,
1818
NATIVE_ONLY_STDERR_LINE,
19+
RENDER_PERTURBATION,
1920
harness_arguments,
2021
perturbation_arguments,
2122
write_broken_ref_project,
@@ -129,6 +130,42 @@ def test_given_perturbed_native_discovery_when_comparing_then_discovery_capture_
129130
assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output
130131

131132

133+
@pytest.mark.parametrize(
134+
"test_case",
135+
[
136+
HarnessRunTestCase(
137+
description="perturbed_native_compile_inputs",
138+
extra_arguments=(),
139+
expected_exit_code=1,
140+
expected_lines=("DIFF project/waffle_shop",),
141+
expected_patterns=(
142+
r"- stage capture 0-compile/002-compile_project_inputs\.json at "
143+
r"/model_inputs/0/macro_deps/",
144+
r"- stage capture 2-plan/002-compile_project_inputs\.json at "
145+
r"/model_inputs/0/macro_deps/",
146+
),
147+
expected_absent=("001-discovered_project_inputs.json",),
148+
)
149+
],
150+
ids=lambda case: case.description,
151+
)
152+
def test_given_perturbed_native_render_when_comparing_then_compile_inputs_capture_names_the_stage(
153+
test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str]
154+
) -> None:
155+
exit_code: int = run_compiler_differential(
156+
harness_arguments(
157+
work_dir=tmp_path / "work",
158+
extra=perturbation_arguments(tmp_path / "perturbation", source=RENDER_PERTURBATION),
159+
)
160+
)
161+
162+
output: str = capsys.readouterr().out
163+
assert exit_code == test_case.expected_exit_code, output
164+
assert all(line in output for line in test_case.expected_lines), output
165+
assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output
166+
assert not any(text in output for text in test_case.expected_absent), output
167+
168+
132169
@pytest.mark.parametrize(
133170
"test_case",
134171
[
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""A real plan captures every compile-input field canonically and proves every render kind."""
2+
3+
from __future__ import annotations
4+
5+
import subprocess
6+
import sys
7+
from pathlib import Path
8+
9+
import pytest
10+
11+
from scripts.compiler_differential._helpers.coverage.render import (
12+
render_capture_problems,
13+
rendered_input_kinds,
14+
required_render_kinds,
15+
)
16+
from scripts.compiler_differential._helpers.running.execution import harness_environment
17+
from scripts.compiler_differential.constants import (
18+
GENERATOR_FEATURE_BLOCKS,
19+
RENDER_STAGE_CAPTURE_SUFFIX,
20+
SQB_ENTRY,
21+
STAGE_CAPTURE_ENV_VAR,
22+
)
23+
from scripts.compiler_differential.main.generate_project import generate_project
24+
from tests.e2e.scripts.compiler_differential._test_types import RenderCaptureTestCase
25+
26+
27+
@pytest.mark.parametrize(
28+
"test_case",
29+
[
30+
RenderCaptureTestCase(
31+
description="every_feature_block_planned",
32+
seed=11,
33+
blocks=GENERATOR_FEATURE_BLOCKS,
34+
command=("plan", "--json"),
35+
expected_kinds=frozenset(required_render_kinds()),
36+
)
37+
],
38+
ids=lambda case: case.description,
39+
)
40+
def test_given_project_with_every_render_input_when_capturing_then_capture_is_complete(
41+
test_case: RenderCaptureTestCase, tmp_path: Path
42+
) -> None:
43+
project_dir: Path = tmp_path / "project"
44+
capture_dir: Path = tmp_path / "captures"
45+
generate_project(seed=test_case.seed, blocks=test_case.blocks).write(project_dir)
46+
47+
completed: subprocess.CompletedProcess[str] = subprocess.run(
48+
[sys.executable, "-c", SQB_ENTRY, "--no-color", *test_case.command],
49+
cwd=project_dir,
50+
env={**harness_environment(), STAGE_CAPTURE_ENV_VAR: str(capture_dir)},
51+
capture_output=True,
52+
text=True,
53+
check=False,
54+
timeout=600,
55+
)
56+
57+
assert completed.returncode == 0, completed.stdout + completed.stderr
58+
text: str = next(capture_dir.glob(f"*{RENDER_STAGE_CAPTURE_SUFFIX}")).read_text(
59+
encoding="utf-8"
60+
)
61+
assert render_capture_problems(text) == ()
62+
assert test_case.expected_kinds <= rendered_input_kinds(text), (
63+
test_case.expected_kinds - rendered_input_kinds(text)
64+
)
65+
66+
67+
if __name__ == "__main__":
68+
pytest.main([__file__, "-vv"])

0 commit comments

Comments
 (0)