Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ Exactly one of a positional target or `--select` is required.
| `--format` | Output format: `tree` (default), `list`, or `json`. |
| `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. |
| `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). |
| `--select`, `-s` | Select resources using standard selector syntax. |
| `--select`, `-s` | Select resources using the same selector syntax as `sqb compile`, `sqb plan` and `sqb build`, for example `path:models/marts`, `tag:finance` or `stg_*`. |
| `--exclude` | Exclude resources from the selection. |

## Model lineage
Expand Down Expand Up @@ -244,7 +244,7 @@ sqb lineage fact_orders --format json --direction upstream

## Depth limiting

`--depth` controls how many hops from the focus node(s) to include. In selector mode, `--depth` requires name, source, seed, or path-between selectors - it cannot be combined with tag or path selectors or comma-intersection selectors.
`--depth` controls how many hops from the focus node(s) to include. In selector mode without `--direction`, `--depth` requires name (including glob patterns), source, seed, or path-between selectors - it cannot be combined with tag or path selectors or comma-intersection selectors.

```bash
# Only immediate parents
Expand Down
336 changes: 44 additions & 292 deletions src/sqlbuild/cli/commands/_helpers/lineage/selection.py

Large diffs are not rendered by default.

13 changes: 0 additions & 13 deletions src/sqlbuild/cli/commands/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, PlaygroundTemplate
from sqlbuild.compiler.lineage.types import ColumnLineageMode
from sqlbuild.compiler.planner.types import SelectorKind
from sqlbuild.spec.contracts.types import WarehouseGroup

C0_CONTROL_CODE_LIMIT: int = 32
Expand Down Expand Up @@ -37,20 +36,8 @@
RULES_SKILLS_ACTION: str = "skills"
SQL_FILE_SUFFIX: str = ".sql"
SELECTOR_INTERSECTION_MARKER: str = ","
PATH_BETWEEN_MARKER: str = "~"
SELECTOR_KIND_SEPARATOR: str = ":"
PATH_SEPARATOR: str = "/"
SELECTOR_EXPANSION_MARKER: str = "+"
COLUMN_TARGET_SEPARATOR: str = "."
COLUMN_TARGET_KIND: str = "column"
SUPPORTED_TYPED_SELECTOR_KINDS: frozenset[str] = frozenset(
{
SelectorKind.SEED,
SelectorKind.SOURCE,
SelectorKind.TAG,
SelectorKind.PATH,
}
)
PLAYGROUND_ORCHESTRATED_PROJECT_DIR: str = "waffle_shop"
PLAYGROUND_ORCHESTRATED_TEMPLATES: frozenset[PlaygroundTemplate] = frozenset(
{PlaygroundTemplate.DAGSTER, PlaygroundTemplate.RIVERS}
Expand Down
20 changes: 0 additions & 20 deletions src/sqlbuild/cli/commands/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -775,26 +775,6 @@ class LineageTarget:
kind_matches: bool


@dataclass(frozen=True)
class ParsedLineageSelector:
"""One parsed non-path lineage selector."""

kind: str
value: str
upstream: bool = False
downstream: bool = False


@dataclass(frozen=True)
class ParsedLineagePathSelector:
"""One parsed path-between lineage selector."""

start_name: str
end_name: str
upstream: bool = False
downstream: bool = False


@dataclass(frozen=True)
class ColumnLineageTrace:
"""Selected column-level lineage trace."""
Expand Down
178 changes: 86 additions & 92 deletions src/sqlbuild/compiler/planner/_helpers/graph/selectors.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,38 +120,28 @@ def resolve_selectors(
if not select and not exclude:
return frozenset(all_keys.values())

selected: set[CompiledObjectKey] = set(all_keys.values()) if not select else set()
raw_select: str
for raw_select in select:
tokens: list[str] = raw_select.split()
token: str
for token in tokens:
resolved: frozenset[CompiledObjectKey] = _resolve_token(
token=token,
all_keys=all_keys,
upstream=upstream,
downstream=downstream,
tag_index=effective_tag_index,
path_index=effective_path_index,
)
selected.update(resolved)

excluded: set[CompiledObjectKey] = set()
raw_exclude: str
for raw_exclude in exclude:
tokens = raw_exclude.split()
for token in tokens:
resolved = _resolve_token(
token=token,
all_keys=all_keys,
upstream=upstream,
downstream=downstream,
tag_index=effective_tag_index,
path_index=effective_path_index,
)
excluded.update(resolved)
selected: frozenset[CompiledObjectKey] = (
resolve_selector_tokens(
selectors=select,
all_keys=all_keys,
upstream=upstream,
downstream=downstream,
tag_index=effective_tag_index,
path_index=effective_path_index,
)
if select
else frozenset(all_keys.values())
)
excluded: frozenset[CompiledObjectKey] = resolve_selector_tokens(
selectors=exclude,
all_keys=all_keys,
upstream=upstream,
downstream=downstream,
tag_index=effective_tag_index,
path_index=effective_path_index,
)

scoped: frozenset[CompiledObjectKey] = frozenset(selected - excluded)
scoped: frozenset[CompiledObjectKey] = selected - excluded
return expand_required_build_resources(
selected_keys=scoped,
upstream=upstream,
Expand All @@ -162,6 +152,60 @@ def resolve_selectors(
)


def resolve_selector_tokens(
*,
selectors: tuple[str, ...],
all_keys: dict[str, CompiledObjectKey],
upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
tag_index: dict[str, frozenset[CompiledObjectKey]],
path_index: dict[CompiledObjectKey, str],
) -> frozenset[CompiledObjectKey]:
"""Union the keys matched by whitespace-separated selector tokens, without build expansion."""

resolved: set[CompiledObjectKey] = set()
raw_selector: str
for raw_selector in selectors:
token: str
for token in raw_selector.split():
resolved.update(
_resolve_token(
token=token,
all_keys=all_keys,
upstream=upstream,
downstream=downstream,
tag_index=tag_index,
path_index=path_index,
)
)
return frozenset(resolved)


def match_selector_keys(
*,
parsed: ParsedSelector,
all_keys: dict[str, CompiledObjectKey],
tag_index: dict[str, frozenset[CompiledObjectKey]],
path_index: dict[CompiledObjectKey, str],
) -> frozenset[CompiledObjectKey]:
"""Return the keys one parsed selector matches before `+` graph expansion."""

if parsed.kind == SelectorKind.TAG:
tagged_keys: frozenset[CompiledObjectKey] = tag_index.get(parsed.value, frozenset())
if not tagged_keys:
raise PlannerInputError(f"no models found with tag '{parsed.value}'", code="S008")
return tagged_keys

if parsed.kind == SelectorKind.PATH:
return _match_path(value=parsed.value, path_index=path_index)

keys: frozenset[CompiledObjectKey] = _lookup_keys(parsed=parsed, all_keys=all_keys)
if not keys:
label: str = "pattern" if _is_name_pattern(parsed.value) else "name"
raise PlannerInputError(f"unknown selector {label} '{parsed.value}'", code="S007")
return keys


def expand_required_build_resources(
*,
selected_keys: frozenset[CompiledObjectKey],
Expand Down Expand Up @@ -272,27 +316,12 @@ def _resolve_single(
result.update(expand_downstream(key=end_key, downstream=downstream))
return frozenset(result)

if parsed.kind == SelectorKind.TAG:
return _resolve_tag(
parsed=parsed,
tag_index=tag_index,
upstream=upstream,
downstream=downstream,
)

if parsed.kind == SelectorKind.PATH:
return _resolve_path(
parsed=parsed,
path_index=path_index,
upstream=upstream,
downstream=downstream,
)

keys: frozenset[CompiledObjectKey] = _lookup_keys(parsed=parsed, all_keys=all_keys)
if not keys:
label: str = "pattern" if _is_name_pattern(parsed.value) else "name"
raise PlannerInputError(f"unknown selector {label} '{parsed.value}'", code="S007")

keys: frozenset[CompiledObjectKey] = match_selector_keys(
parsed=parsed,
all_keys=all_keys,
tag_index=tag_index,
path_index=path_index,
)
result: set[CompiledObjectKey] = set(keys)
key: CompiledObjectKey
if parsed.upstream:
Expand All @@ -304,40 +333,14 @@ def _resolve_single(
return frozenset(result)


def _resolve_tag(
*,
parsed: ParsedSelector,
tag_index: dict[str, frozenset[CompiledObjectKey]],
upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
) -> frozenset[CompiledObjectKey]:
"""Resolve a tag selector to matching keys with optional graph expansion."""

tagged_keys: frozenset[CompiledObjectKey] = tag_index.get(parsed.value, frozenset())
if not tagged_keys:
raise PlannerInputError(f"no models found with tag '{parsed.value}'", code="S008")

result: set[CompiledObjectKey] = set(tagged_keys)
key: CompiledObjectKey
if parsed.upstream:
for key in tagged_keys:
result.update(expand_upstream(key=key, upstream=upstream))
if parsed.downstream:
for key in tagged_keys:
result.update(expand_downstream(key=key, downstream=downstream))
return frozenset(result)


def _resolve_path(
def _match_path(
*,
parsed: ParsedSelector,
value: str,
path_index: dict[CompiledObjectKey, str],
upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
) -> frozenset[CompiledObjectKey]:
"""Resolve a path selector to matching keys with optional graph expansion."""
"""Match a `models/`-rooted path selector against model folders."""

folder: str = _normalize_path_selector_value(parsed.value)
folder: str = _normalize_path_selector_value(value)
selector_folder: str = _model_path_candidate(folder)
matched_keys: frozenset[CompiledObjectKey] = frozenset(
key
Expand All @@ -354,16 +357,7 @@ def _resolve_path(
),
code="S009",
)

result: set[CompiledObjectKey] = set(matched_keys)
key: CompiledObjectKey
if parsed.upstream:
for key in matched_keys:
result.update(expand_upstream(key=key, upstream=upstream))
if parsed.downstream:
for key in matched_keys:
result.update(expand_downstream(key=key, downstream=downstream))
return frozenset(result)
return matched_keys


def _normalize_path_selector_value(value: str) -> str:
Expand Down
24 changes: 24 additions & 0 deletions src/sqlbuild/compiler/planner/main/selection/selector_match.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
"""Public entrypoint matching one parsed selector before graph expansion."""

from __future__ import annotations

from sqlbuild.compiler.compile.models import CompiledObjectKey
from sqlbuild.compiler.planner._helpers.graph.selectors import match_selector_keys
from sqlbuild.compiler.planner.models import ParsedSelector


def match_project_selector(
*,
parsed: ParsedSelector,
all_keys: dict[str, CompiledObjectKey],
tag_index: dict[str, frozenset[CompiledObjectKey]],
path_index: dict[CompiledObjectKey, str],
) -> frozenset[CompiledObjectKey]:
"""Return the keys one parsed selector matches before `+` graph expansion."""

return match_selector_keys(
parsed=parsed,
all_keys=all_keys,
tag_index=tag_index,
path_index=path_index,
)
27 changes: 27 additions & 0 deletions src/sqlbuild/compiler/planner/main/selection/selector_tokens.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
"""Public selector-token resolution entrypoint without required build resources."""

from __future__ import annotations

from sqlbuild.compiler.compile.models import CompiledObjectKey
from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_selector_tokens


def resolve_project_selector_tokens(
*,
selectors: tuple[str, ...],
all_keys: dict[str, CompiledObjectKey],
upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]],
tag_index: dict[str, frozenset[CompiledObjectKey]],
path_index: dict[CompiledObjectKey, str],
) -> frozenset[CompiledObjectKey]:
"""Resolve selector strings to the keys they match, without adding required build resources."""

return resolve_selector_tokens(
selectors=selectors,
all_keys=all_keys,
upstream=upstream_deps,
downstream=downstream_deps,
tag_index=tag_index,
path_index=path_index,
)
16 changes: 16 additions & 0 deletions tests/e2e/src/sqlbuild/cli/commands/main/lineage/_test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,19 @@ class DiamondLineageTreeCliTestCase:
command: tuple[str, ...]
expected_expanded_names: tuple[str, ...]
expected_max_lines: int


@dataclass(frozen=True)
class SharedSelectorCliTestCase:
description: str
selector: str
expected_node_ids: tuple[str, ...]
expected_selected_models: int
expected_selected_functions: int


@dataclass(frozen=True)
class SharedSelectorErrorCliTestCase:
description: str
selector: str
expected_fragment: str
Loading
Loading