fix(test): reject mocks reached through reference calls that read referencing helpers

A mock a check reaches through a reference call such as __source("orders"),
directly or from a helper the check reads, is now checked like a mock read by
CTE name, so one that reads a helper calling a reference fails compile with a
located P013 instead of failing at run time. The mock-only P013 now points at
the mocked model's __ref() call, and its help says to call __ref() when the
check reads the mock by CTE name.

Author: kvlonge

src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py

@@ -251,28 +251,36 @@ def report_test_without_target_model(
         for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads)
         if graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES)
     ]
-    first_call: str | None = None
+    first_calls: dict[SqlReferenceKind, tuple[str, str]] = {}
+    called_models: set[str] = set()
     for cte in _reader_ctes(payload):
         for reference in _references(sql=cte.sql_body, syntax=syntax):
             kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind)
             if reference.ref_name not in mocked.get(kind, frozenset()):
                 continue
             read_mocks.append(f"{kind.fixture_cte_prefix}{reference.ref_name}")
-            if first_call is None and kind is not SqlReferenceKind.TABLE_FUNCTION:
-                first_call = _reference_call(reference)
+            if kind is SqlReferenceKind.REF:
+                called_models.add(reference.ref_name)
+            if kind is not SqlReferenceKind.TABLE_FUNCTION:
+                first_calls.setdefault(kind, (cte.name, _reference_call(reference)))
     mocks: tuple[str, ...] = tuple(dict.fromkeys(read_mocks))
     mocked_models: tuple[str, ...] = tuple(
         mock.removeprefix(REF_TEST_CTE_PREFIX)
         for mock in mocks
         if mock.startswith(REF_TEST_CTE_PREFIX)
     )
     reader: CompileSqlTestCte | None = next(iter(_reader_ctes(payload)), None)
+    located: tuple[str, str] | None = first_calls.get(SqlReferenceKind.REF) or (
+        _mock_name_read(readers=_reader_ctes(payload), mocks=mocks)
+        or next(iter(first_calls.values()), None)
+    )
     test_name: str = test_block.name or test_file.relative_path.stem
     message: str
     help_text: str
     if mocked_models:
         plural: str = "s" if len(mocked_models) > 1 else ""
         owner: str = "models'" if plural else "model's"
+        verb: str = "keep" if called_models.issuperset(mocked_models) else "call"
         mocked_ctes: str = ", ".join(f"{REF_TEST_CTE_PREFIX}{name}" for name in mocked_models)
         calls: str = " and ".join(
             SqlReferenceKind.REF.example_call(name, quote='"') for name in mocked_models
@@ -283,7 +291,7 @@ def report_test_without_target_model(
         )
         help_text = (
             f"Mock the {owner} inputs instead (for example __source__<source> or "
-            f"__ref__<upstream model>) and keep {calls} in the __assert__ or __expected__ CTE, "
+            f"__ref__<upstream model>) and {verb} {calls} in the __assert__ or __expected__ CTE, "
             "or remove the test."
         )
     else:
@@ -294,13 +302,27 @@ def report_test_without_target_model(
             "runs that model with its mocks, or remove the test."
         )
     _HelperDiagnostics(test_file=test_file, test_block=test_block).report(
-        cte_name=reader.name if reader is not None else test_name,
-        call=first_call,
+        cte_name=located[0] if located else (reader.name if reader is not None else test_name),
+        call=located[1] if located else None,
         message=message,
         help=help_text,
     )
 
 
+def _mock_name_read(
+    *, readers: tuple[CompileSqlTestCte, ...], mocks: tuple[str, ...]
+) -> tuple[str, str] | None:
+    """Return the first check reading a mocked model by CTE name, with that name."""
+
+    for mock in mocks:
+        if not mock.startswith(REF_TEST_CTE_PREFIX):
+            continue
+        for reader in readers:
+            if re.search(rf"(?<![\w$]){re.escape(mock)}(?![\w$])", reader.sql_body, re.IGNORECASE):
+                return reader.name, mock
+    return None
+
+
 def report_mocks_reading_referencing_helpers(
     *,
     authored_ctes: tuple[CompileSqlTestCte, ...],
@@ -333,8 +355,18 @@ def report_mocks_reading_referencing_helpers(
     graph: SqlTestCteGraph = sql_test_cte_graph(
         authored_ctes=authored_ctes, reader_ctes=reader_ctes
     )
+    called_mocks: list[str] = []
+    for sql in (
+        *(cte.sql_body for cte in reader_ctes),
+        *(graph.ctes[key].sql_body for key in _read_helper_keys(graph)),
+    ):
+        for reference in _references(sql=sql, syntax=syntax):
+            called_mocks.extend(
+                _mock_keys(kind=SqlReferenceKind(reference.ref_kind), reference=reference)
+            )
     used_mocks: tuple[str, ...] = _used_mock_keys(
         graph=graph,
+        called_mocks=tuple(called_mocks),
         model_references={
             model_input.model_file.file_path.stem: model_input.references
             for model_input in model_inputs
@@ -386,17 +418,19 @@ def _read_helper_keys(graph: SqlTestCteGraph) -> tuple[str, ...]:
 def _used_mock_keys(
     *,
     graph: SqlTestCteGraph,
+    called_mocks: tuple[str, ...],
     model_references: dict[str, tuple[CompileSqlReference, ...]],
     target_model_names: tuple[str, ...],
     mocked: frozenset[str],
 ) -> tuple[str, ...]:
-    """Return mocks the test's readers or the models it runs read, in a stable order."""
+    """Return mocks the test's checks and read helpers, or the models it runs, read in order."""
 
     used: list[str] = [
         key
         for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads)
         if graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES)
     ]
+    used.extend(key for key in dict.fromkeys(called_mocks) if key in graph.ctes and key not in used)
     pending: list[str] = [name for name in target_model_names if name not in mocked]
     visited: set[str] = set(pending)
     while pending:

tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py

@@ -1849,6 +1849,39 @@ def prepare_optional_ceremony_project(*, tmp_path: Path, test_files: dict[str, s
         '  SELECT item_id, amount_doubled FROM __ref("item_archive")\n'
         ")\n"
     ),
+    "mock_called_reads_referencing_helper": (
+        'extras AS (SELECT item_id, amount FROM __ref("item_extras")),\n'
+        "mid AS (SELECT item_id, amount FROM extras),\n"
+        "__ref__items AS (SELECT 7 AS item_id, 5 AS amount),\n"
+        "__source__item_returns AS (SELECT item_id FROM mid),\n"
+        "__expected__item_totals AS (SELECT 7 AS item_id, 10 AS amount_doubled),\n"
+        "__assert__no_returns AS (\n"
+        '  SELECT item_id FROM __source("item_returns") WHERE item_id <> 7\n'
+        ")\n"
+    ),
+    "mock_called_reads_referencing_helper_directly": (
+        'extras AS (SELECT item_id, amount FROM __ref("item_extras")),\n'
+        "__ref__items AS (SELECT 7 AS item_id, 5 AS amount),\n"
+        "__source__item_returns AS (SELECT item_id FROM extras),\n"
+        "__expected__item_totals AS (SELECT 7 AS item_id, 10 AS amount_doubled),\n"
+        "__assert__no_returns AS (\n"
+        '  SELECT item_id FROM __source("item_returns") WHERE item_id <> 7\n'
+        ")\n"
+    ),
+    "assertion_reads_mocked_source_and_model": (
+        "__ref__items AS (\n  SELECT 7 AS item_id, 5 AS amount\n),\n"
+        "__source__item_returns AS (SELECT 7 AS item_id),\n"
+        "__assert__returned_items AS (\n"
+        '  SELECT r.item_id FROM __source("item_returns") AS r\n'
+        '  JOIN __ref("items") AS i USING (item_id) WHERE i.amount <> 5\n'
+        ")\n"
+    ),
+    "assertion_reads_mock_by_name": (
+        "__ref__items AS (\n  SELECT 7 AS item_id, 5 AS amount\n),\n"
+        "__assert__only_mocked_items AS (\n"
+        "  SELECT item_id FROM __ref__items WHERE item_id <> 7\n"
+        ")\n"
+    ),
     "mock_reads_referencing_helper": (
         "__ref__items AS (\n  SELECT item_id, 10 AS amount FROM base_rows\n),\n"
         'base_rows AS (SELECT item_id FROM __ref("item_totals")),\n'

tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py

@@ -706,6 +706,168 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m
             expected_column=39,
             expected_help_fragment='__ref("<model>")',
         ),
+        HelperReferenceErrorE2ETestCase(
+            description="a mock a check calls reading a referencing helper through a middle helper on the python engine",
+            test_name="mock_called_reads_referencing_helper",
+            engine="python",
+            expected_code="P013",
+            expected_message=(
+                "SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
+                '__ref("item_extras"); mocks and fixtures are defined before the models the '
+                "test runs, so the helper cannot be resolved for them"
+            ),
+            expected_line=4,
+            expected_column=40,
+            expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="a mock a check calls reading a referencing helper directly on the python engine",
+            test_name="mock_called_reads_referencing_helper_directly",
+            engine="python",
+            expected_code="P013",
+            expected_message=(
+                "SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
+                '__ref("item_extras"); mocks and fixtures are defined before the models the '
+                "test runs, so the helper cannot be resolved for them"
+            ),
+            expected_line=4,
+            expected_column=40,
+            expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="test reading a mocked source and a mocked model on the python engine",
+            test_name="assertion_reads_mocked_source_and_model",
+            engine="python",
+            expected_code="P013",
+            expected_message=(
+                "SQL test 'assertion_reads_mocked_source_and_model' mocks the model it tests "
+                "(__ref__items), so the test has no model to run against"
+            ),
+            expected_line=10,
+            expected_column=8,
+            expected_help_fragment='and keep __ref("items") in the __assert__ or __expected__ CTE',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="test reading a mocked model by CTE name on the python engine",
+            test_name="assertion_reads_mock_by_name",
+            engine="python",
+            expected_code="P013",
+            expected_message=(
+                "SQL test 'assertion_reads_mock_by_name' mocks the model it tests (__ref__items), "
+                "so the test has no model to run against"
+            ),
+            expected_line=8,
+            expected_column=23,
+            expected_help_fragment='and call __ref("items") in the __assert__ or __expected__ CTE',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="a mock a check calls reading a referencing helper through a middle helper on the native engine",
+            test_name="mock_called_reads_referencing_helper",
+            engine="native",
+            expected_code="P013",
+            expected_message=(
+                "SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
+                '__ref("item_extras"); mocks and fixtures are defined before the models the '
+                "test runs, so the helper cannot be resolved for them"
+            ),
+            expected_line=4,
+            expected_column=40,
+            expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="a mock a check calls reading a referencing helper directly on the native engine",
+            test_name="mock_called_reads_referencing_helper_directly",
+            engine="native",
+            expected_code="P013",
+            expected_message=(
+                "SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
+                '__ref("item_extras"); mocks and fixtures are defined before the models the '
+                "test runs, so the helper cannot be resolved for them"
+            ),
+            expected_line=4,
+            expected_column=40,
+            expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="test reading a mocked source and a mocked model on the native engine",
+            test_name="assertion_reads_mocked_source_and_model",
+            engine="native",
+            expected_code="P013",
+            expected_message=(
+                "SQL test 'assertion_reads_mocked_source_and_model' mocks the model it tests "
+                "(__ref__items), so the test has no model to run against"
+            ),
+            expected_line=10,
+            expected_column=8,
+            expected_help_fragment='and keep __ref("items") in the __assert__ or __expected__ CTE',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="test reading a mocked model by CTE name on the native engine",
+            test_name="assertion_reads_mock_by_name",
+            engine="native",
+            expected_code="P013",
+            expected_message=(
+                "SQL test 'assertion_reads_mock_by_name' mocks the model it tests (__ref__items), "
+                "so the test has no model to run against"
+            ),
+            expected_line=8,
+            expected_column=23,
+            expected_help_fragment='and call __ref("items") in the __assert__ or __expected__ CTE',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="a mock a check calls reading a referencing helper through a middle helper on the native-preview engine",
+            test_name="mock_called_reads_referencing_helper",
+            engine="native-preview",
+            expected_code="P013",
+            expected_message=(
+                "SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
+                '__ref("item_extras"); mocks and fixtures are defined before the models the '
+                "test runs, so the helper cannot be resolved for them"
+            ),
+            expected_line=4,
+            expected_column=40,
+            expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="a mock a check calls reading a referencing helper directly on the native-preview engine",
+            test_name="mock_called_reads_referencing_helper_directly",
+            engine="native-preview",
+            expected_code="P013",
+            expected_message=(
+                "SQL test mock '__source__item_returns' reads helper CTE 'extras', which calls "
+                '__ref("item_extras"); mocks and fixtures are defined before the models the '
+                "test runs, so the helper cannot be resolved for them"
+            ),
+            expected_line=4,
+            expected_column=40,
+            expected_help_fragment='FROM __ref__item_extras rather than FROM __ref("item_extras")',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="test reading a mocked source and a mocked model on the native-preview engine",
+            test_name="assertion_reads_mocked_source_and_model",
+            engine="native-preview",
+            expected_code="P013",
+            expected_message=(
+                "SQL test 'assertion_reads_mocked_source_and_model' mocks the model it tests "
+                "(__ref__items), so the test has no model to run against"
+            ),
+            expected_line=10,
+            expected_column=8,
+            expected_help_fragment='and keep __ref("items") in the __assert__ or __expected__ CTE',
+        ),
+        HelperReferenceErrorE2ETestCase(
+            description="test reading a mocked model by CTE name on the native-preview engine",
+            test_name="assertion_reads_mock_by_name",
+            engine="native-preview",
+            expected_code="P013",
+            expected_message=(
+                "SQL test 'assertion_reads_mock_by_name' mocks the model it tests (__ref__items), "
+                "so the test has no model to run against"
+            ),
+            expected_line=8,
+            expected_column=23,
+            expected_help_fragment='and call __ref("items") in the __assert__ or __expected__ CTE',
+        ),
     ),
     ids=lambda case: case.description,
 )