Skip to content

fix(test): reject mocks reached through reference calls that read referencing helpers #1035

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kvlonge merged 1 commit into from
Oct 8, 2026
Merged

fix(test): reject mocks reached through reference calls that read referencing helpers #1035

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 41 additions & 7 deletions src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,28 +251,36 @@ def report_test_without_target_model(
for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads)
if graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES)
]
first_call: str | None = None
first_calls: dict[SqlReferenceKind, tuple[str, str]] = {}
called_models: set[str] = set()
for cte in _reader_ctes(payload):
for reference in _references(sql=cte.sql_body, syntax=syntax):
kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind)
if reference.ref_name not in mocked.get(kind, frozenset()):
continue
read_mocks.append(f"{kind.fixture_cte_prefix}{reference.ref_name}")
if first_call is None and kind is not SqlReferenceKind.TABLE_FUNCTION:
first_call = _reference_call(reference)
if kind is SqlReferenceKind.REF:
called_models.add(reference.ref_name)
if kind is not SqlReferenceKind.TABLE_FUNCTION:
first_calls.setdefault(kind, (cte.name, _reference_call(reference)))
mocks: tuple[str, ...] = tuple(dict.fromkeys(read_mocks))
mocked_models: tuple[str, ...] = tuple(
mock.removeprefix(REF_TEST_CTE_PREFIX)
for mock in mocks
if mock.startswith(REF_TEST_CTE_PREFIX)
)
reader: CompileSqlTestCte | None = next(iter(_reader_ctes(payload)), None)
located: tuple[str, str] | None = first_calls.get(SqlReferenceKind.REF) or (
_mock_name_read(readers=_reader_ctes(payload), mocks=mocks)
or next(iter(first_calls.values()), None)
)
test_name: str = test_block.name or test_file.relative_path.stem
message: str
help_text: str
if mocked_models:
plural: str = "s" if len(mocked_models) > 1 else ""
owner: str = "models'" if plural else "model's"
verb: str = "keep" if called_models.issuperset(mocked_models) else "call"
mocked_ctes: str = ", ".join(f"{REF_TEST_CTE_PREFIX}{name}" for name in mocked_models)
calls: str = " and ".join(
SqlReferenceKind.REF.example_call(name, quote='"') for name in mocked_models
Expand All @@ -283,7 +291,7 @@ def report_test_without_target_model(
)
help_text = (
f"Mock the {owner} inputs instead (for example __source__<source> or "
f"__ref__<upstream model>) and keep {calls} in the __assert__ or __expected__ CTE, "
f"__ref__<upstream model>) and {verb} {calls} in the __assert__ or __expected__ CTE, "
"or remove the test."
)
else:
Expand All @@ -294,13 +302,27 @@ def report_test_without_target_model(
"runs that model with its mocks, or remove the test."
)
_HelperDiagnostics(test_file=test_file, test_block=test_block).report(
cte_name=reader.name if reader is not None else test_name,
call=first_call,
cte_name=located[0] if located else (reader.name if reader is not None else test_name),
call=located[1] if located else None,
message=message,
help=help_text,
)


def _mock_name_read(
*, readers: tuple[CompileSqlTestCte, ...], mocks: tuple[str, ...]
) -> tuple[str, str] | None:
"""Return the first check reading a mocked model by CTE name, with that name."""

for mock in mocks:
if not mock.startswith(REF_TEST_CTE_PREFIX):
continue
for reader in readers:
if re.search(rf"(?<![\w$]){re.escape(mock)}(?![\w$])", reader.sql_body, re.IGNORECASE):
return reader.name, mock
return None


def report_mocks_reading_referencing_helpers(
*,
authored_ctes: tuple[CompileSqlTestCte, ...],
Expand Down Expand Up @@ -333,8 +355,18 @@ def report_mocks_reading_referencing_helpers(
graph: SqlTestCteGraph = sql_test_cte_graph(
authored_ctes=authored_ctes, reader_ctes=reader_ctes
)
called_mocks: list[str] = []
for sql in (
*(cte.sql_body for cte in reader_ctes),
*(graph.ctes[key].sql_body for key in _read_helper_keys(graph)),
):
for reference in _references(sql=sql, syntax=syntax):
called_mocks.extend(
_mock_keys(kind=SqlReferenceKind(reference.ref_kind), reference=reference)
)
used_mocks: tuple[str, ...] = _used_mock_keys(
graph=graph,
called_mocks=tuple(called_mocks),
model_references={
model_input.model_file.file_path.stem: model_input.references
for model_input in model_inputs
Expand Down Expand Up @@ -386,17 +418,19 @@ def _read_helper_keys(graph: SqlTestCteGraph) -> tuple[str, ...]:
def _used_mock_keys(
*,
graph: SqlTestCteGraph,
called_mocks: tuple[str, ...],
model_references: dict[str, tuple[CompileSqlReference, ...]],
target_model_names: tuple[str, ...],
mocked: frozenset[str],
) -> tuple[str, ...]:
"""Return mocks the test's readers or the models it runs read, in a stable order."""
"""Return mocks the test's checks and read helpers, or the models it runs, read in order."""

used: list[str] = [
key
for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads)
if graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES)
]
used.extend(key for key in dict.fromkeys(called_mocks) if key in graph.ctes and key not in used)
pending: list[str] = [name for name in target_model_names if name not in mocked]
visited: set[str] = set(pending)
while pending:
Expand Down
33 changes: 33 additions & 0 deletions tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1849,6 +1849,39 @@ def prepare_optional_ceremony_project(*, tmp_path: Path, test_files: dict[str, s
' SELECT item_id, amount_doubled FROM __ref("item_archive")\n'
")\n"
),
"mock_called_reads_referencing_helper": (
'extras AS (SELECT item_id, amount FROM __ref("item_extras")),\n'
"mid AS (SELECT item_id, amount FROM extras),\n"
"__ref__items AS (SELECT 7 AS item_id, 5 AS amount),\n"
"__source__item_returns AS (SELECT item_id FROM mid),\n"
"__expected__item_totals AS (SELECT 7 AS item_id, 10 AS amount_doubled),\n"
"__assert__no_returns AS (\n"
' SELECT item_id FROM __source("item_returns") WHERE item_id <> 7\n'
")\n"
),
"mock_called_reads_referencing_helper_directly": (
'extras AS (SELECT item_id, amount FROM __ref("item_extras")),\n'
"__ref__items AS (SELECT 7 AS item_id, 5 AS amount),\n"
"__source__item_returns AS (SELECT item_id FROM extras),\n"
"__expected__item_totals AS (SELECT 7 AS item_id, 10 AS amount_doubled),\n"
"__assert__no_returns AS (\n"
' SELECT item_id FROM __source("item_returns") WHERE item_id <> 7\n'
")\n"
),
"assertion_reads_mocked_source_and_model": (
"__ref__items AS (\n SELECT 7 AS item_id, 5 AS amount\n),\n"
"__source__item_returns AS (SELECT 7 AS item_id),\n"
"__assert__returned_items AS (\n"
' SELECT r.item_id FROM __source("item_returns") AS r\n'
' JOIN __ref("items") AS i USING (item_id) WHERE i.amount <> 5\n'
")\n"
),
"assertion_reads_mock_by_name": (
"__ref__items AS (\n SELECT 7 AS item_id, 5 AS amount\n),\n"
"__assert__only_mocked_items AS (\n"
" SELECT item_id FROM __ref__items WHERE item_id <> 7\n"
")\n"
),
"mock_reads_referencing_helper": (
"__ref__items AS (\n SELECT item_id, 10 AS amount FROM base_rows\n),\n"
'base_rows AS (SELECT item_id FROM __ref("item_totals")),\n'
Expand Down
162 changes: 162 additions & 0 deletions tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,168 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m
expected_column=39,
expected_help_fragment='__ref("<model>")',
),
HelperReferenceErrorE2ETestCase(
description="a mock a check calls reading a referencing helper through a middle helper on the python engine",
test_name="mock_called_reads_referencing_helper",
engine="python",
expected_code="P013",
expected_message=(
"SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
'__ref("item_extras"); mocks and fixtures are defined before the models the '
"test runs, so the helper cannot be resolved for them"
),
expected_line=4,
expected_column=40,
expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
),
HelperReferenceErrorE2ETestCase(
description="a mock a check calls reading a referencing helper directly on the python engine",
test_name="mock_called_reads_referencing_helper_directly",
engine="python",
expected_code="P013",
expected_message=(
"SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
'__ref("item_extras"); mocks and fixtures are defined before the models the '
"test runs, so the helper cannot be resolved for them"
),
expected_line=4,
expected_column=40,
expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
),
HelperReferenceErrorE2ETestCase(
description="test reading a mocked source and a mocked model on the python engine",
test_name="assertion_reads_mocked_source_and_model",
engine="python",
expected_code="P013",
expected_message=(
"SQL test 'assertion_reads_mocked_source_and_model' mocks the model it tests "
"(__ref__items), so the test has no model to run against"
),
expected_line=10,
expected_column=8,
expected_help_fragment='and keep __ref("items") in the __assert__ or __expected__ CTE',
),
HelperReferenceErrorE2ETestCase(
description="test reading a mocked model by CTE name on the python engine",
test_name="assertion_reads_mock_by_name",
engine="python",
expected_code="P013",
expected_message=(
"SQL test 'assertion_reads_mock_by_name' mocks the model it tests (__ref__items), "
"so the test has no model to run against"
),
expected_line=8,
expected_column=23,
expected_help_fragment='and call __ref("items") in the __assert__ or __expected__ CTE',
),
HelperReferenceErrorE2ETestCase(
description="a mock a check calls reading a referencing helper through a middle helper on the native engine",
test_name="mock_called_reads_referencing_helper",
engine="native",
expected_code="P013",
expected_message=(
"SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
'__ref("item_extras"); mocks and fixtures are defined before the models the '
"test runs, so the helper cannot be resolved for them"
),
expected_line=4,
expected_column=40,
expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
),
HelperReferenceErrorE2ETestCase(
description="a mock a check calls reading a referencing helper directly on the native engine",
test_name="mock_called_reads_referencing_helper_directly",
engine="native",
expected_code="P013",
expected_message=(
"SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
'__ref("item_extras"); mocks and fixtures are defined before the models the '
"test runs, so the helper cannot be resolved for them"
),
expected_line=4,
expected_column=40,
expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
),
HelperReferenceErrorE2ETestCase(
description="test reading a mocked source and a mocked model on the native engine",
test_name="assertion_reads_mocked_source_and_model",
engine="native",
expected_code="P013",
expected_message=(
"SQL test 'assertion_reads_mocked_source_and_model' mocks the model it tests "
"(__ref__items), so the test has no model to run against"
),
expected_line=10,
expected_column=8,
expected_help_fragment='and keep __ref("items") in the __assert__ or __expected__ CTE',
),
HelperReferenceErrorE2ETestCase(
description="test reading a mocked model by CTE name on the native engine",
test_name="assertion_reads_mock_by_name",
engine="native",
expected_code="P013",
expected_message=(
"SQL test 'assertion_reads_mock_by_name' mocks the model it tests (__ref__items), "
"so the test has no model to run against"
),
expected_line=8,
expected_column=23,
expected_help_fragment='and call __ref("items") in the __assert__ or __expected__ CTE',
),
HelperReferenceErrorE2ETestCase(
description="a mock a check calls reading a referencing helper through a middle helper on the native-preview engine",
test_name="mock_called_reads_referencing_helper",
engine="native-preview",
expected_code="P013",
expected_message=(
"SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
'__ref("item_extras"); mocks and fixtures are defined before the models the '
"test runs, so the helper cannot be resolved for them"
),
expected_line=4,
expected_column=40,
expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
),
HelperReferenceErrorE2ETestCase(
description="a mock a check calls reading a referencing helper directly on the native-preview engine",
test_name="mock_called_reads_referencing_helper_directly",
engine="native-preview",
expected_code="P013",
expected_message=(
"SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
'__ref("item_extras"); mocks and fixtures are defined before the models the '
"test runs, so the helper cannot be resolved for them"
),
expected_line=4,
expected_column=40,
expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
),
HelperReferenceErrorE2ETestCase(
description="test reading a mocked source and a mocked model on the native-preview engine",
test_name="assertion_reads_mocked_source_and_model",
engine="native-preview",
expected_code="P013",
expected_message=(
"SQL test 'assertion_reads_mocked_source_and_model' mocks the model it tests "
"(__ref__items), so the test has no model to run against"
),
expected_line=10,
expected_column=8,
expected_help_fragment='and keep __ref("items") in the __assert__ or __expected__ CTE',
),
HelperReferenceErrorE2ETestCase(
description="test reading a mocked model by CTE name on the native-preview engine",
test_name="assertion_reads_mock_by_name",
engine="native-preview",
expected_code="P013",
expected_message=(
"SQL test 'assertion_reads_mock_by_name' mocks the model it tests (__ref__items), "
"so the test has no model to run against"
),
expected_line=8,
expected_column=23,
expected_help_fragment='and call __ref("items") in the __assert__ or __expected__ CTE',
),
),
ids=lambda case: case.description,
)
Expand Down
Loading