Skip to content

Commit 306eceb

Browse files
authored
fix: make sqb lineage use the shared project selector parser (#972)
sqb lineage --select/--exclude now resolve through the same selector implementation as compile, plan and build, so path:models/... selectors, name globs and shared error codes behave identically across commands. The undocumented lineage-only path:<folder> form without the models/ root is now rejected with S012, matching every other command.
1 parent 73bcecb commit 306eceb

12 files changed

Lines changed: 516 additions & 426 deletions

File tree

‎src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ Exactly one of a positional target or `--select` is required.
4343
| `--format` | Output format: `tree` (default), `list`, or `json`. |
4444
| `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. |
4545
| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). |
46-
| `--select`, `-s` | Select resources using standard selector syntax. |
46+
| `--select`, `-s` | Select resources using the same selector syntax as `sqb compile`, `sqb plan` and `sqb build`, for example `path:models/marts`, `tag:finance` or `stg_*`. |
4747
| `--exclude` | Exclude resources from the selection. |
4848

4949
## Model lineage
@@ -244,7 +244,7 @@ sqb lineage fact_orders --format json --direction upstream
244244

245245
## Depth limiting
246246

247-
`--depth` controls how many hops from the focus node(s) to include. In selector mode, `--depth` requires name, source, seed, or path-between selectors - it cannot be combined with tag or path selectors or comma-intersection selectors.
247+
`--depth` controls how many hops from the focus node(s) to include. In selector mode without `--direction`, `--depth` requires name (including glob patterns), source, seed, or path-between selectors - it cannot be combined with tag or path selectors or comma-intersection selectors.
248248

249249
```bash
250250
# Only immediate parents

‎src/sqlbuild/cli/commands/_helpers/lineage/selection.py‎

Lines changed: 44 additions & 292 deletions
Large diffs are not rendered by default.

‎src/sqlbuild/cli/commands/constants.py‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, PlaygroundTemplate
66
from sqlbuild.compiler.lineage.types import ColumnLineageMode
7-
from sqlbuild.compiler.planner.types import SelectorKind
87
from sqlbuild.spec.contracts.types import WarehouseGroup
98

109
C0_CONTROL_CODE_LIMIT: int = 32
@@ -37,20 +36,8 @@
3736
RULES_SKILLS_ACTION: str = "skills"
3837
SQL_FILE_SUFFIX: str = ".sql"
3938
SELECTOR_INTERSECTION_MARKER: str = ","
40-
PATH_BETWEEN_MARKER: str = "~"
41-
SELECTOR_KIND_SEPARATOR: str = ":"
42-
PATH_SEPARATOR: str = "/"
43-
SELECTOR_EXPANSION_MARKER: str = "+"
4439
COLUMN_TARGET_SEPARATOR: str = "."
4540
COLUMN_TARGET_KIND: str = "column"
46-
SUPPORTED_TYPED_SELECTOR_KINDS: frozenset[str] = frozenset(
47-
{
48-
SelectorKind.SEED,
49-
SelectorKind.SOURCE,
50-
SelectorKind.TAG,
51-
SelectorKind.PATH,
52-
}
53-
)
5441
PLAYGROUND_ORCHESTRATED_PROJECT_DIR: str = "waffle_shop"
5542
PLAYGROUND_ORCHESTRATED_TEMPLATES: frozenset[PlaygroundTemplate] = frozenset(
5643
{PlaygroundTemplate.DAGSTER, PlaygroundTemplate.RIVERS}

‎src/sqlbuild/cli/commands/models.py‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -775,26 +775,6 @@ class LineageTarget:
775775
kind_matches: bool
776776

777777

778-
@dataclass(frozen=True)
779-
class ParsedLineageSelector:
780-
"""One parsed non-path lineage selector."""
781-
782-
kind: str
783-
value: str
784-
upstream: bool = False
785-
downstream: bool = False
786-
787-
788-
@dataclass(frozen=True)
789-
class ParsedLineagePathSelector:
790-
"""One parsed path-between lineage selector."""
791-
792-
start_name: str
793-
end_name: str
794-
upstream: bool = False
795-
downstream: bool = False
796-
797-
798778
@dataclass(frozen=True)
799779
class ColumnLineageTrace:
800780
"""Selected column-level lineage trace."""

‎src/sqlbuild/compiler/planner/_helpers/graph/selectors.py‎

Lines changed: 86 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -120,38 +120,28 @@ def resolve_selectors(
120120
if not select and not exclude:
121121
return frozenset(all_keys.values())
122122

123-
selected: set[CompiledObjectKey] = set(all_keys.values()) if not select else set()
124-
raw_select: str
125-
for raw_select in select:
126-
tokens: list[str] = raw_select.split()
127-
token: str
128-
for token in tokens:
129-
resolved: frozenset[CompiledObjectKey] = _resolve_token(
130-
token=token,
131-
all_keys=all_keys,
132-
upstream=upstream,
133-
downstream=downstream,
134-
tag_index=effective_tag_index,
135-
path_index=effective_path_index,
136-
)
137-
selected.update(resolved)
138-
139-
excluded: set[CompiledObjectKey] = set()
140-
raw_exclude: str
141-
for raw_exclude in exclude:
142-
tokens = raw_exclude.split()
143-
for token in tokens:
144-
resolved = _resolve_token(
145-
token=token,
146-
all_keys=all_keys,
147-
upstream=upstream,
148-
downstream=downstream,
149-
tag_index=effective_tag_index,
150-
path_index=effective_path_index,
151-
)
152-
excluded.update(resolved)
123+
selected: frozenset[CompiledObjectKey] = (
124+
resolve_selector_tokens(
125+
selectors=select,
126+
all_keys=all_keys,
127+
upstream=upstream,
128+
downstream=downstream,
129+
tag_index=effective_tag_index,
130+
path_index=effective_path_index,
131+
)
132+
if select
133+
else frozenset(all_keys.values())
134+
)
135+
excluded: frozenset[CompiledObjectKey] = resolve_selector_tokens(
136+
selectors=exclude,
137+
all_keys=all_keys,
138+
upstream=upstream,
139+
downstream=downstream,
140+
tag_index=effective_tag_index,
141+
path_index=effective_path_index,
142+
)
153143

154-
scoped: frozenset[CompiledObjectKey] = frozenset(selected - excluded)
144+
scoped: frozenset[CompiledObjectKey] = selected - excluded
155145
return expand_required_build_resources(
156146
selected_keys=scoped,
157147
upstream=upstream,
@@ -162,6 +152,60 @@ def resolve_selectors(
162152
)
163153

164154

155+
def resolve_selector_tokens(
156+
*,
157+
selectors: tuple[str, ...],
158+
all_keys: dict[str, CompiledObjectKey],
159+
upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
160+
downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
161+
tag_index: dict[str, frozenset[CompiledObjectKey]],
162+
path_index: dict[CompiledObjectKey, str],
163+
) -> frozenset[CompiledObjectKey]:
164+
"""Union the keys matched by whitespace-separated selector tokens, without build expansion."""
165+
166+
resolved: set[CompiledObjectKey] = set()
167+
raw_selector: str
168+
for raw_selector in selectors:
169+
token: str
170+
for token in raw_selector.split():
171+
resolved.update(
172+
_resolve_token(
173+
token=token,
174+
all_keys=all_keys,
175+
upstream=upstream,
176+
downstream=downstream,
177+
tag_index=tag_index,
178+
path_index=path_index,
179+
)
180+
)
181+
return frozenset(resolved)
182+
183+
184+
def match_selector_keys(
185+
*,
186+
parsed: ParsedSelector,
187+
all_keys: dict[str, CompiledObjectKey],
188+
tag_index: dict[str, frozenset[CompiledObjectKey]],
189+
path_index: dict[CompiledObjectKey, str],
190+
) -> frozenset[CompiledObjectKey]:
191+
"""Return the keys one parsed selector matches before `+` graph expansion."""
192+
193+
if parsed.kind == SelectorKind.TAG:
194+
tagged_keys: frozenset[CompiledObjectKey] = tag_index.get(parsed.value, frozenset())
195+
if not tagged_keys:
196+
raise PlannerInputError(f"no models found with tag '{parsed.value}'", code="S008")
197+
return tagged_keys
198+
199+
if parsed.kind == SelectorKind.PATH:
200+
return _match_path(value=parsed.value, path_index=path_index)
201+
202+
keys: frozenset[CompiledObjectKey] = _lookup_keys(parsed=parsed, all_keys=all_keys)
203+
if not keys:
204+
label: str = "pattern" if _is_name_pattern(parsed.value) else "name"
205+
raise PlannerInputError(f"unknown selector {label} '{parsed.value}'", code="S007")
206+
return keys
207+
208+
165209
def expand_required_build_resources(
166210
*,
167211
selected_keys: frozenset[CompiledObjectKey],
@@ -272,27 +316,12 @@ def _resolve_single(
272316
result.update(expand_downstream(key=end_key, downstream=downstream))
273317
return frozenset(result)
274318

275-
if parsed.kind == SelectorKind.TAG:
276-
return _resolve_tag(
277-
parsed=parsed,
278-
tag_index=tag_index,
279-
upstream=upstream,
280-
downstream=downstream,
281-
)
282-
283-
if parsed.kind == SelectorKind.PATH:
284-
return _resolve_path(
285-
parsed=parsed,
286-
path_index=path_index,
287-
upstream=upstream,
288-
downstream=downstream,
289-
)
290-
291-
keys: frozenset[CompiledObjectKey] = _lookup_keys(parsed=parsed, all_keys=all_keys)
292-
if not keys:
293-
label: str = "pattern" if _is_name_pattern(parsed.value) else "name"
294-
raise PlannerInputError(f"unknown selector {label} '{parsed.value}'", code="S007")
295-
319+
keys: frozenset[CompiledObjectKey] = match_selector_keys(
320+
parsed=parsed,
321+
all_keys=all_keys,
322+
tag_index=tag_index,
323+
path_index=path_index,
324+
)
296325
result: set[CompiledObjectKey] = set(keys)
297326
key: CompiledObjectKey
298327
if parsed.upstream:
@@ -304,40 +333,14 @@ def _resolve_single(
304333
return frozenset(result)
305334

306335

307-
def _resolve_tag(
308-
*,
309-
parsed: ParsedSelector,
310-
tag_index: dict[str, frozenset[CompiledObjectKey]],
311-
upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
312-
downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
313-
) -> frozenset[CompiledObjectKey]:
314-
"""Resolve a tag selector to matching keys with optional graph expansion."""
315-
316-
tagged_keys: frozenset[CompiledObjectKey] = tag_index.get(parsed.value, frozenset())
317-
if not tagged_keys:
318-
raise PlannerInputError(f"no models found with tag '{parsed.value}'", code="S008")
319-
320-
result: set[CompiledObjectKey] = set(tagged_keys)
321-
key: CompiledObjectKey
322-
if parsed.upstream:
323-
for key in tagged_keys:
324-
result.update(expand_upstream(key=key, upstream=upstream))
325-
if parsed.downstream:
326-
for key in tagged_keys:
327-
result.update(expand_downstream(key=key, downstream=downstream))
328-
return frozenset(result)
329-
330-
331-
def _resolve_path(
336+
def _match_path(
332337
*,
333-
parsed: ParsedSelector,
338+
value: str,
334339
path_index: dict[CompiledObjectKey, str],
335-
upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
336-
downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
337340
) -> frozenset[CompiledObjectKey]:
338-
"""Resolve a path selector to matching keys with optional graph expansion."""
341+
"""Match a `models/`-rooted path selector against model folders."""
339342

340-
folder: str = _normalize_path_selector_value(parsed.value)
343+
folder: str = _normalize_path_selector_value(value)
341344
selector_folder: str = _model_path_candidate(folder)
342345
matched_keys: frozenset[CompiledObjectKey] = frozenset(
343346
key
@@ -354,16 +357,7 @@ def _resolve_path(
354357
),
355358
code="S009",
356359
)
357-
358-
result: set[CompiledObjectKey] = set(matched_keys)
359-
key: CompiledObjectKey
360-
if parsed.upstream:
361-
for key in matched_keys:
362-
result.update(expand_upstream(key=key, upstream=upstream))
363-
if parsed.downstream:
364-
for key in matched_keys:
365-
result.update(expand_downstream(key=key, downstream=downstream))
366-
return frozenset(result)
360+
return matched_keys
367361

368362

369363
def _normalize_path_selector_value(value: str) -> str:
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
"""Public entrypoint matching one parsed selector before graph expansion."""
2+
3+
from __future__ import annotations
4+
5+
from sqlbuild.compiler.compile.models import CompiledObjectKey
6+
from sqlbuild.compiler.planner._helpers.graph.selectors import match_selector_keys
7+
from sqlbuild.compiler.planner.models import ParsedSelector
8+
9+
10+
def match_project_selector(
11+
*,
12+
parsed: ParsedSelector,
13+
all_keys: dict[str, CompiledObjectKey],
14+
tag_index: dict[str, frozenset[CompiledObjectKey]],
15+
path_index: dict[CompiledObjectKey, str],
16+
) -> frozenset[CompiledObjectKey]:
17+
"""Return the keys one parsed selector matches before `+` graph expansion."""
18+
19+
return match_selector_keys(
20+
parsed=parsed,
21+
all_keys=all_keys,
22+
tag_index=tag_index,
23+
path_index=path_index,
24+
)
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
"""Public selector-token resolution entrypoint without required build resources."""
2+
3+
from __future__ import annotations
4+
5+
from sqlbuild.compiler.compile.models import CompiledObjectKey
6+
from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_selector_tokens
7+
8+
9+
def resolve_project_selector_tokens(
10+
*,
11+
selectors: tuple[str, ...],
12+
all_keys: dict[str, CompiledObjectKey],
13+
upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
14+
downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
15+
tag_index: dict[str, frozenset[CompiledObjectKey]],
16+
path_index: dict[CompiledObjectKey, str],
17+
) -> frozenset[CompiledObjectKey]:
18+
"""Resolve selector strings to the keys they match, without adding required build resources."""
19+
20+
return resolve_selector_tokens(
21+
selectors=selectors,
22+
all_keys=all_keys,
23+
upstream=upstream_deps,
24+
downstream=downstream_deps,
25+
tag_index=tag_index,
26+
path_index=path_index,
27+
)

‎tests/e2e/src/sqlbuild/cli/commands/main/lineage/_test_types.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,19 @@ class DiamondLineageTreeCliTestCase:
3939
command: tuple[str, ...]
4040
expected_expanded_names: tuple[str, ...]
4141
expected_max_lines: int
42+
43+
44+
@dataclass(frozen=True)
45+
class SharedSelectorCliTestCase:
46+
description: str
47+
selector: str
48+
expected_node_ids: tuple[str, ...]
49+
expected_selected_models: int
50+
expected_selected_functions: int
51+
52+
53+
@dataclass(frozen=True)
54+
class SharedSelectorErrorCliTestCase:
55+
description: str
56+
selector: str
57+
expected_fragment: str

0 commit comments

Comments
 (0)